From 0a69fca6574d421c3bdbc97f120b33b5e2e8946e Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 14 Nov 2019 23:15:51 -0600 Subject: [PATCH 1/3] allow for translate store race in test (by using retry) In this case, the test is reading from the translateStore replica before the translateStore replication has had time to deliver its log to the replica. The only way to truly address this in the translate store would be to route all key misses that happen on a read-only replica to the primary translate store (or somehow know when the primary is done sending to replicas) for actual verification that the key does not exist. That's more involved than we want to do here; this PR just addresses the problem in the test. --- api_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/api_test.go b/api_test.go index fafae614f..d412def1a 100644 --- a/api_test.go +++ b/api_test.go @@ -175,10 +175,15 @@ func TestAPI_Import(t *testing.T) { } // Query node1. - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if err := test.RetryUntil(5*time.Second, func() error { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + return err + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { + return fmt.Errorf("unexpected column ids: %+v", columns) + } + return nil + }); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - t.Fatalf("unexpected column ids: %+v", columns) } }) } From ed82a535e542aed22016d4d3a749ec092a3ed7aa Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 18 Nov 2019 11:39:50 -0600 Subject: [PATCH 2/3] Fix ranked cache logic to support reducing cached values below the threshold. Prior to this commit, if a cache value was reduced to a value that fell below the threshold, the operation would be ignored and the cached value would remain at the old, higher value. This commit also fixes logic which reduces a cached value within the framework of uint64 values by subracting the absolute value of the negative value (since adding a negitive doesn't work with unsigned integers). --- cache.go | 2 ++ cache_test.go | 27 ++++++++++++++++++++++++--- fragment.go | 11 ++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cache.go b/cache.go index de836b8c8..05572fa11 100644 --- a/cache.go +++ b/cache.go @@ -172,6 +172,7 @@ func (c *rankCache) Add(id uint64, n uint64) { // unless the count is 0, which is effectively used // to clear the cache value. if n < c.thresholdValue && n > 0 { + delete(c.entries, id) return } @@ -185,6 +186,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() if n < c.thresholdValue { + delete(c.entries, id) return } diff --git a/cache_test.go b/cache_test.go index de1bd3a0b..d4c0982e9 100644 --- a/cache_test.go +++ b/cache_test.go @@ -20,8 +20,8 @@ import ( "github.com/pilosa/pilosa/v2" ) -// Ensure a bitmap query can be executed. -func TestCache_Rank(t *testing.T) { +// Ensure cache stays constrained to its configured size. +func TestCache_Rank_Size(t *testing.T) { cacheSize := uint32(3) cache := pilosa.NewRankCache(cacheSize) for i := 1; i < int(2*cacheSize); i++ { @@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) { if cache.Len() != int(cacheSize) { t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize) } - +} + +// Ensure cache entries set below threshold are handled appropriately. +func TestCache_Rank_Threshold(t *testing.T) { + cacheSize := uint32(5) + cache := pilosa.NewRankCache(cacheSize) + for i := 1; i < int(2*cacheSize); i++ { + cache.Add(uint64(i), 3) + } + + // Set the cache value for rows 4 and 5 to a number below the threshold + // value (which is 3), and ensure that they gets zeroed out. + cache.Add(4, 1) + cache.BulkAdd(5, 1) + cache.Recalculate() + + if cache.Get(4) != 0 { + t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0) + } + if cache.Get(5) != 0 { + t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0) + } } diff --git a/fragment.go b/fragment.go index aa65574c6..e6c714937 100644 --- a/fragment.go +++ b/fragment.go @@ -2099,7 +2099,16 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e f.rowCache.Add(rowID, nil) if updateCache { anyChanged = true - f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes)) + if changes < 0 { + absChanges := uint64(-1 * changes) + if absChanges <= f.cache.Get(rowID) { + f.cache.BulkAdd(rowID, f.cache.Get(rowID)-absChanges) + } else { + f.cache.BulkAdd(rowID, 0) + } + } else { + f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes)) + } } } // we only set this if we need to update the cache From 58182ff56354cba9133613ee535bfb82ab5e844e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 4 Nov 2019 11:21:14 -0600 Subject: [PATCH 3/3] Pilosa Enterprise private CI --- .circleci/config.yml | 37 +++++++++++++++++++++++++++++-------- Dockerfile | 5 ++++- Makefile | 7 ++++++- go.mod | 2 +- go.sum | 2 +- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5384c556d..fb2c70908 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,10 +8,13 @@ defaults: &defaults fast-checkout: &fast-checkout attach_workspace: at: . +add-github-auth: &add-github-auth + run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com" jobs: setup: <<: *defaults steps: + - *add-github-auth - checkout - restore_cache: keys: @@ -28,11 +31,13 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make check-license-headers linter: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0 - run: sudo cp bin/golangci-lint /usr/local/bin/ - run: make golangci-lint @@ -40,6 +45,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make build GOOS=linux GOARCH=arm GOARM=5 - run: make build GOOS=linux GOARCH=arm GOARM=6 - run: make build GOOS=linux GOARCH=arm GOARM=7 @@ -48,18 +54,21 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test test-golang-1.13-shard22: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test SHARD_WIDTH=22 test-golang-1.13-race: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: command: make test TESTFLAGS="-race -v -timeout=30m" @@ -73,6 +82,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test ENTERPRISE=1 test-golang-1.12: @@ -81,6 +91,7 @@ jobs: - image: circleci/golang:1.12 steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test test-golang-1.11: @@ -89,18 +100,21 @@ jobs: - image: circleci/golang:1.11 steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test cluster-tests: <<: *defaults steps: - *fast-checkout + - *add-github-auth - setup_remote_docker - run: make clustertests-build prerelease: <<: *base-test steps: - *fast-checkout + - *add-github-auth - run: make prerelease - store_artifacts: path: build @@ -111,6 +125,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make release - store_artifacts: path: build @@ -123,6 +138,7 @@ jobs: steps: - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout + - *add-github-auth - run: sudo pip install awscli - run: make prerelease-upload dockerhub-upload: @@ -130,11 +146,23 @@ jobs: steps: - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout + - *add-github-auth - setup_remote_docker - run: make docker - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: docker push pilosa/pilosa:master + docker-enterprise-upload: + <<: *defaults + steps: + #- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR + - *fast-checkout + - *add-github-auth + - setup_remote_docker + - run: make docker-enterprise + - run: docker tag pilosa:$(git describe --tags) molecula.azurecr.io/pilosa-enterprise:unstable + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push molecula.azurecr.io/pilosa-enterprise:unstable workflows: version: 2 test: @@ -185,11 +213,4 @@ workflows: only: /^v.*/ branches: ignore: /.*/ - - prerelease-upload: - requires: - - prerelease - - dockerhub-upload: - requires: - - linter - - check-license-headers - - test-golang-1.13 + diff --git a/Dockerfile b/Dockerfile index 0dcf9c4a1..1bcabbdab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,11 @@ FROM golang:1.13.0 as builder +ARG BUILD_FLAGS +ARG MAKE_FLAGS + COPY . pilosa -RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a" +RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a ${BUILD_FLAGS}" ${MAKE_FLAGS} FROM alpine:3.9.4 diff --git a/Makefile b/Makefile index a25195ef8..bee802120 100644 --- a/Makefile +++ b/Makefile @@ -128,9 +128,14 @@ generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" . + docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) +# Create Docker image from Dockerfile (enterprise) +docker-enterprise: + docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" . + @echo Created docker image: pilosa-enterprise:$(VERSION) + # Compile Pilosa inside Docker container docker-build: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa diff --git a/go.mod b/go.mod index ef21e563a..eb38d9288 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect diff --git a/go.sum b/go.sum index dd930d309..6607cf98d 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -98,7 +99,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=