Merge branch 'enterprise' into sqdeadlock

2026-09-07 00:55:55 +00:00 · 2019-11-18 20:51:40 -06:00 · 2019-11-18 20:51:40 -06:00 · 01309e4fc5
commit 01309e4fc5
parent f204b37760 d3c8728821
9 changed files with 85 additions and 19 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -8,10 +8,13 @@ defaults: &defaults
 fast-checkout: &fast-checkout
  attach_workspace:
    at: .
+add-github-auth: &add-github-auth
+  run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com"
 jobs:
  setup:
    <<: *defaults
    steps:
+      - *add-github-auth
      - checkout
      - restore_cache:
          keys:
@ -28,11 +31,13 @@ jobs:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: make check-license-headers
  linter:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0
      - run: sudo cp bin/golangci-lint /usr/local/bin/
      - run: make golangci-lint
@ -40,6 +45,7 @@ jobs:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: make build GOOS=linux GOARCH=arm GOARM=5
      - run: make build GOOS=linux GOARCH=arm GOARM=6
      - run: make build GOOS=linux GOARCH=arm GOARM=7
@ -48,18 +54,21 @@ jobs:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run: make test
  test-golang-1.13-shard22:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run: make test SHARD_WIDTH=22
  test-golang-1.13-race:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run:
          command: make test TESTFLAGS="-race -v -timeout=30m"
@ -73,6 +82,7 @@ jobs:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run: make test ENTERPRISE=1
  test-golang-1.12:
@ -81,6 +91,7 @@ jobs:
      - image: circleci/golang:1.12
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run: make test
  test-golang-1.11:
@ -89,18 +100,21 @@ jobs:
      - image: circleci/golang:1.11
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: sudo apt-get install lsof
      - run: make test
  cluster-tests:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - setup_remote_docker
      - run: make clustertests-build
  prerelease:
    <<: *base-test
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: make prerelease
      - store_artifacts:
          path: build
@ -111,6 +125,7 @@ jobs:
    <<: *defaults
    steps:
      - *fast-checkout
+      - *add-github-auth
      - run: make release
      - store_artifacts:
          path: build
@ -123,6 +138,7 @@ jobs:
    steps:
      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
      - *fast-checkout
+      - *add-github-auth
      - run: sudo pip install awscli
      - run: make prerelease-upload
  dockerhub-upload:
@ -130,11 +146,23 @@ jobs:
    steps:
      - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
      - *fast-checkout
+      - *add-github-auth
      - setup_remote_docker
      - run: make docker
      - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
      - run: docker push pilosa/pilosa:master
+  docker-enterprise-upload:
+    <<: *defaults
+    steps:
+      #- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
+      - *fast-checkout
+      - *add-github-auth
+      - setup_remote_docker
+      - run: make docker-enterprise
+      - run: docker tag pilosa:$(git describe --tags) molecula.azurecr.io/pilosa-enterprise:unstable
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: docker push molecula.azurecr.io/pilosa-enterprise:unstable
 workflows:
  version: 2
  test:
@ -185,11 +213,4 @@ workflows:
              only: /^v.*/
            branches:
              ignore: /.*/
-      - prerelease-upload:
-          requires:
-            - prerelease
-      - dockerhub-upload:
-          requires:
-            - linter
-            - check-license-headers
-            - test-golang-1.13
+
--- a/5
+++ b/5
@ -1,8 +1,11 @@
 FROM golang:1.13.0 as builder

+ARG BUILD_FLAGS
+ARG MAKE_FLAGS
+
 COPY . pilosa

-RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a"
+RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a ${BUILD_FLAGS}" ${MAKE_FLAGS}

 FROM alpine:3.9.4

--- a/7
+++ b/7
@ -128,9 +128,14 @@ generate: generate-protoc generate-stringer generate-pql

 # Create Docker image from Dockerfile
 docker:
-	docker build -t "pilosa:$(VERSION)" .
+	docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
 	@echo Created docker image: pilosa:$(VERSION)

+# Create Docker image from Dockerfile (enterprise)
+docker-enterprise:
+	docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" .
+	@echo Created docker image: pilosa-enterprise:$(VERSION)
+
 # Compile Pilosa inside Docker container
 docker-build:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
--- a/api_test.go
+++ b/api_test.go
@ -175,10 +175,15 @@ func TestAPI_Import(t *testing.T) {
 		}

 		// Query node1.
-		if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		if err := test.RetryUntil(5*time.Second, func() error {
+			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+				return err
+			} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
+				return fmt.Errorf("unexpected column ids: %+v", columns)
+			}
+			return nil
+		}); err != nil {
 			t.Fatal(err)
-		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) {
-			t.Fatalf("unexpected column ids: %+v", columns)
 		}
 	})
 }
--- a/cache.go
+++ b/cache.go
@ -172,6 +172,7 @@ func (c *rankCache) Add(id uint64, n uint64) {
 	// unless the count is 0, which is effectively used
 	// to clear the cache value.
 	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}

@ -185,6 +186,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}

--- a/cache_test.go
+++ b/cache_test.go
@ -20,8 +20,8 @@ import (
 	"github.com/pilosa/pilosa/v2"
 )

-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@ -31,5 +31,26 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
 }
--- a/fragment.go
+++ b/fragment.go
@ -2099,7 +2099,16 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e
 		f.rowCache.Add(rowID, nil)
 		if updateCache {
 			anyChanged = true
-			f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes))
+			if changes < 0 {
+				absChanges := uint64(-1 * changes)
+				if absChanges <= f.cache.Get(rowID) {
+					f.cache.BulkAdd(rowID, f.cache.Get(rowID)-absChanges)
+				} else {
+					f.cache.BulkAdd(rowID, 0)
+				}
+			} else {
+				f.cache.BulkAdd(rowID, f.cache.Get(rowID)+uint64(changes))
+			}
 		}
 	}
 	// we only set this if we need to update the cache
--- a/go.mod
+++ b/go.mod
@ -36,7 +36,7 @@ require (
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
-	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
+	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
 	golang.org/x/text v0.3.2 // indirect
--- a/go.sum
+++ b/go.sum
@ -41,6 +41,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@ -98,7 +99,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=