diff --git a/.circleci/config.yml b/.circleci/config.yml index 5384c556d..fb2c70908 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,10 +8,13 @@ defaults: &defaults fast-checkout: &fast-checkout attach_workspace: at: . +add-github-auth: &add-github-auth + run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com" jobs: setup: <<: *defaults steps: + - *add-github-auth - checkout - restore_cache: keys: @@ -28,11 +31,13 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make check-license-headers linter: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0 - run: sudo cp bin/golangci-lint /usr/local/bin/ - run: make golangci-lint @@ -40,6 +45,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make build GOOS=linux GOARCH=arm GOARM=5 - run: make build GOOS=linux GOARCH=arm GOARM=6 - run: make build GOOS=linux GOARCH=arm GOARM=7 @@ -48,18 +54,21 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test test-golang-1.13-shard22: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test SHARD_WIDTH=22 test-golang-1.13-race: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: command: make test TESTFLAGS="-race -v -timeout=30m" @@ -73,6 +82,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test ENTERPRISE=1 test-golang-1.12: @@ -81,6 +91,7 @@ jobs: - image: circleci/golang:1.12 steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test test-golang-1.11: @@ -89,18 +100,21 @@ jobs: - image: circleci/golang:1.11 steps: - *fast-checkout + - *add-github-auth - run: sudo apt-get install lsof - run: make test cluster-tests: <<: *defaults steps: - *fast-checkout + - *add-github-auth - setup_remote_docker - run: make clustertests-build prerelease: <<: *base-test steps: - *fast-checkout + - *add-github-auth - run: make prerelease - store_artifacts: path: build @@ -111,6 +125,7 @@ jobs: <<: *defaults steps: - *fast-checkout + - *add-github-auth - run: make release - store_artifacts: path: build @@ -123,6 +138,7 @@ jobs: steps: - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout + - *add-github-auth - run: sudo pip install awscli - run: make prerelease-upload dockerhub-upload: @@ -130,11 +146,23 @@ jobs: steps: - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout + - *add-github-auth - setup_remote_docker - run: make docker - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: docker push pilosa/pilosa:master + docker-enterprise-upload: + <<: *defaults + steps: + #- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR + - *fast-checkout + - *add-github-auth + - setup_remote_docker + - run: make docker-enterprise + - run: docker tag pilosa:$(git describe --tags) molecula.azurecr.io/pilosa-enterprise:unstable + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push molecula.azurecr.io/pilosa-enterprise:unstable workflows: version: 2 test: @@ -185,11 +213,4 @@ workflows: only: /^v.*/ branches: ignore: /.*/ - - prerelease-upload: - requires: - - prerelease - - dockerhub-upload: - requires: - - linter - - check-license-headers - - test-golang-1.13 + diff --git a/Dockerfile b/Dockerfile index 0dcf9c4a1..1bcabbdab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,11 @@ FROM golang:1.13.0 as builder +ARG BUILD_FLAGS +ARG MAKE_FLAGS + COPY . pilosa -RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a" +RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a ${BUILD_FLAGS}" ${MAKE_FLAGS} FROM alpine:3.9.4 diff --git a/Makefile b/Makefile index a25195ef8..bee802120 100644 --- a/Makefile +++ b/Makefile @@ -128,9 +128,14 @@ generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" . + docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) +# Create Docker image from Dockerfile (enterprise) +docker-enterprise: + docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" . + @echo Created docker image: pilosa-enterprise:$(VERSION) + # Compile Pilosa inside Docker container docker-build: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa diff --git a/api_test.go b/api_test.go index fafae614f..d412def1a 100644 --- a/api_test.go +++ b/api_test.go @@ -175,10 +175,15 @@ func TestAPI_Import(t *testing.T) { } // Query node1. - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if err := test.RetryUntil(5*time.Second, func() error { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + return err + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { + return fmt.Errorf("unexpected column ids: %+v", columns) + } + return nil + }); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - t.Fatalf("unexpected column ids: %+v", columns) } }) } diff --git a/cache.go b/cache.go index de836b8c8..05572fa11 100644 --- a/cache.go +++ b/cache.go @@ -172,6 +172,7 @@ func (c *rankCache) Add(id uint64, n uint64) { // unless the count is 0, which is effectively used // to clear the cache value. if n < c.thresholdValue && n > 0 { + delete(c.entries, id) return } @@ -185,6 +186,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() if n < c.thresholdValue { + delete(c.entries, id) return } diff --git a/cache_test.go b/cache_test.go index de1bd3a0b..d4c0982e9 100644 --- a/cache_test.go +++ b/cache_test.go @@ -20,8 +20,8 @@ import ( "github.com/pilosa/pilosa/v2" ) -// Ensure a bitmap query can be executed. -func TestCache_Rank(t *testing.T) { +// Ensure cache stays constrained to its configured size. +func TestCache_Rank_Size(t *testing.T) { cacheSize := uint32(3) cache := pilosa.NewRankCache(cacheSize) for i := 1; i < int(2*cacheSize); i++ { @@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) { if cache.Len() != int(cacheSize) { t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize) } - +} + +// Ensure cache entries set below threshold are handled appropriately. +func TestCache_Rank_Threshold(t *testing.T) { + cacheSize := uint32(5) + cache := pilosa.NewRankCache(cacheSize) + for i := 1; i < int(2*cacheSize); i++ { + cache.Add(uint64(i), 3) + } + + // Set the cache value for rows 4 and 5 to a number below the threshold + // value (which is 3), and ensure that they gets zeroed out. + cache.Add(4, 1) + cache.BulkAdd(5, 1) + cache.Recalculate() + + if cache.Get(4) != 0 { + t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0) + } + if cache.Get(5) != 0 { + t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0) + } } diff --git a/fragment.go b/fragment.go index aa65574c6..e6c714937 100644 --- a/fragment.go +++ b/fragment.go @@ -2099,7 +2099,16 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e f.rowCache.Add(rowID, nil) if updateCache { anyChanged = true - f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes)) + if changes < 0 { + absChanges := uint64(-1 * changes) + if absChanges <= f.cache.Get(rowID) { + f.cache.BulkAdd(rowID, f.cache.Get(rowID)-absChanges) + } else { + f.cache.BulkAdd(rowID, 0) + } + } else { + f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes)) + } } } // we only set this if we need to update the cache diff --git a/go.mod b/go.mod index ef21e563a..eb38d9288 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect diff --git a/go.sum b/go.sum index dd930d309..6607cf98d 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -98,7 +99,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=