add support to modify shard width at build time

use "make <x> SHARD_WIDTH=nn" fix tests to run and pass at different shardwidths add shardwidth22 test to circle ci
2026-08-28 02:44:59 +00:00 · 2019-03-29 17:39:30 -05:00 · 2019-03-29 17:39:30 -05:00 · 836b467d3d
commit 836b467d3d
parent 927e8b8942
11 changed files with 51 additions and 30 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -44,6 +44,12 @@ jobs:
      - *fast-checkout
      - run: sudo apt-get install lsof
      - run: make test
+  test-golang-1.12-shard22: &base-test
+    <<: *defaults
+    steps:
+      - *fast-checkout
+      - run: sudo apt-get install lsof
+      - run: make test SHARD_WIDTH=22
  test-golang-1.12-race:
    <<: *defaults
    steps:
--- a/4
+++ b/4
@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
+SHARD_WIDTH = 20
 LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
 GO_VERSION=latest
 ENTERPRISE ?= 0
@ -14,6 +15,7 @@ RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
 BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
+BUILD_TAGS += shardwidth$(SHARD_WIDTH)
 export GO111MODULE=on

 # Run tests and compile Pilosa
@ -29,7 +31,7 @@ vendor: go.mod

 # Run test suite
 test:
-	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
+	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) 

 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 1300000, nil)
+	_, err = field.SetBit(1, ShardWidth+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 2600000, nil)
+	_, err = field.SetBit(1, ShardWidth*2+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = field.SetBit(1, 3900000, nil)
+	_, err = field.SetBit(1, ShardWidth*3+1, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@ -755,7 +755,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 			t.Fatal(err)
 		}
 		tc.SetBit("i", "f", 1, 101, nil)
-		tc.SetBit("i", "f", 1, 1300000, nil)
+		tc.SetBit("i", "f", 1, ShardWidth+1, nil)

 		// Before starting the resize, get the CheckSum to use for
 		// comparison later.
--- a/executor_test.go
+++ b/executor_test.go
@ -2226,11 +2226,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
 	})

 	t.Run("Remote SetBit", func(t *testing.T) {
-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil {
-			t.Fatalf("quuerying remote: %v", err)
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil {
+			t.Fatalf("querying remote: %v", err)
 		}

-		if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) {
+		if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) {
 			t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns())
 		}
 	})
@ -2241,11 +2241,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
 			t.Fatalf("creating field: %v", err)
 		}

-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil {
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil {
 			t.Fatalf("quuerying remote: %v", err)
 		}

-		if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) {
+		if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) {
 			t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns())
 		}
 	})
--- a/fragment.go
+++ b/fragment.go
@ -48,8 +48,9 @@ import (

 const (
 	// ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16.
-	shardWidthExponent = 20
-	ShardWidth         = 1 << shardWidthExponent
+	// shardWidthExponent = 20 // set in shardwidthNN.go files
+
+	ShardWidth = 1 << shardWidthExponent

 	// shardVsContainerExponent is the power of 2 of ShardWith minus the power
 	// of two of roaring container width (which is 16).
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@ -137,14 +137,14 @@ func TestFragment_SetRow(t *testing.T) {
 	rowID := uint64(1000)

 	// Set bits on the fragment.
-	if _, err := f.setBit(rowID, 8000001); err != nil {
+	if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil {
 		t.Fatal(err)
-	} else if _, err := f.setBit(rowID, 8065536); err != nil {
+	} else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil {
 		t.Fatal(err)
 	}

 	// Verify data on row.
-	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
+	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) {
 		t.Fatalf("unexpected columns: %+v", cols)
 	}
 	// Verify count on row.
@ -153,7 +153,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}

 	// Set row (overwrite existing data).
-	row := NewRow(8000002, 8065537, 8131074)
+	row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000)
 	if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
 		t.Fatal(err)
 	} else if !changed {
@ -161,7 +161,7 @@ func TestFragment_SetRow(t *testing.T) {
 	}

 	// Verify data on row.
-	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
+	if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) {
 		t.Fatalf("unexpected columns after set row: %+v", cols)
 	}
 	// Verify count on row.
@ -1914,7 +1914,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 	f := mustOpenFragment("i", "f", viewStandard, 0, "")
 	defer f.Clean(b)
 	// Generate some intersecting data.
-	maxX := 1048576 / 2
+	maxX := ShardWidth / 2
 	sz := maxX
 	rows := make([]uint64, sz)
 	cols := make([]uint64, sz)
@ -1950,7 +1950,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {

 func BenchmarkFragment_Import(b *testing.B) {
 	b.StopTimer()
-	maxX := 1048576 * 5 * 2
+	maxX := ShardWidth * 5 * 2
 	sz := maxX
 	rows := make([]uint64, sz)
 	cols := make([]uint64, sz)
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@ -299,10 +299,10 @@ func TestBitmap_Max(t *testing.T) {

 // Ensure CountRange is correct even if rangekey is prior to initial container.
 func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
-	s := uint64(2009 * 1048576)
-	e := uint64(2010 * 1048576)
+	s := uint64(2009 * pilosa.ShardWidth)
+	e := uint64(2010 * pilosa.ShardWidth)

-	start := s + (39314024 % 1048576)
+	start := s + (39314024 % pilosa.ShardWidth)
 	bm0 := roaring.NewFileBitmap()
 	for i := uint64(0); i < 65536; i++ {
 		if (i+1)%4096 == 0 {
--- a/server/cluster_test.go
+++ b/server/cluster_test.go
@ -85,10 +85,10 @@ func TestMain_SendReceiveMessage(t *testing.T) {
 	}

 	// Write data on first node.
-	if _, err := m0.Query("i", "", `
+	if _, err := m0.Query("i", "", fmt.Sprintf(`
            Set(1, f=1)
-            Set(2400000, f=1)
-        `); err != nil {
+            Set(%d, f=1)
+        `, 2*pilosa.ShardWidth+1)); err != nil {
 		t.Fatal(err)
 	}

--- a/server/handler_test.go
+++ b/server/handler_test.go
@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	gohttp "net/http"
@ -109,8 +110,8 @@ func TestHandler_Endpoints(t *testing.T) {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		}
 		body := w.Body.String()
-		target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}
-`
+		target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}
+`, pilosa.ShardWidth)
 		if body != target {
 			t.Fatalf("%s != %s", target, body)
 		}
@ -294,7 +295,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
-		} else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" {
+		} else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" {
 			t.Fatalf("unexpected body: %s", body)
 		}
 	})
@ -311,10 +312,11 @@ func TestHandler_Endpoints(t *testing.T) {
 	t.Run("ColumnAttrs_JSON", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
+		exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n"
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
-		} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" {
-			t.Fatalf("unexpected body: %s", body)
+		} else if body := w.Body.String(); body != exp {
+			t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp)
 		}
 	})

--- a/shardwidth20.go
+++ b/shardwidth20.go
@ -0,0 +1,5 @@
+// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32
+
+package pilosa
+
+const shardWidthExponent = 20
--- a/shardwidth22.go
+++ b/shardwidth22.go
@ -0,0 +1,5 @@
+// +build shardwidth22
+
+package pilosa
+
+const shardWidthExponent = 22