diff --git a/.circleci/config.yml b/.circleci/config.yml index d31f1255e..ebbff8ea2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: parameters: version: type: string - default: "1.15.5" + default: "1.15.8" resource_class: type: string default: medium @@ -17,7 +17,8 @@ executors: commands: add-github-auth: steps: - - run: git config --global url."https://moleculacorp:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com".insteadOf "https://github.com" + - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/" + - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:" restore-mod-cache: steps: - restore_cache: @@ -67,7 +68,7 @@ jobs: name: golang steps: - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request - - run: curl https://moleculacorp:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e + - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/pilosa/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e test-build-arm: executor: name: golang @@ -84,7 +85,7 @@ jobs: default: medium golang_version: type: string - default: "1.15.5" + default: "1.15.8" shard_width: type: string default: "20" @@ -114,17 +115,6 @@ jobs: - checkout-plus - setup_remote_docker - run: make clustertests-build - prerelease: - executor: - name: golang - steps: - - checkout-plus - - run: make prerelease - - store_artifacts: - path: build - - persist_to_workspace: - root: . - paths: build release: executor: name: golang @@ -132,48 +122,54 @@ jobs: - checkout-plus - attach_workspace: at: . - - run: make release + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin + - run: make docker-release - store_artifacts: path: build - persist_to_workspace: root: . paths: build - prerelease-upload: - docker: - - image: circleci/python:2.7-jessie + publish_release: + executor: + name: golang steps: - - checkout-plus - attach_workspace: at: . - - run: sudo pip install awscli - - run: make prerelease-upload + - run: go get github.com/tcnksm/ghr + - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/ docker-build: executor: name: golang steps: - checkout-plus - - setup_remote_docker - - run: make docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin + - run: make docker GO_VERSION=1.15.8 - run: docker run pilosa:$(git describe --tags) help dockerhub-upload-unstable: executor: name: golang steps: - checkout-plus - - setup_remote_docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >> dockerhub-upload-stable: executor: name: golang steps: - checkout-plus - - setup_remote_docker + - setup_remote_docker: + version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 + - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - run: docker run pilosa:$(git describe --tags) help - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >> - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest @@ -186,12 +182,15 @@ workflows: tags: only: /^v.*/ - linter: + context: molecula requires: - setup - check-license-headers: + context: molecula requires: - setup - go-mod-tidy: + context: molecula requires: - setup - check-changelog-label: @@ -199,6 +198,7 @@ workflows: requires: - setup - test-build-arm: + context: molecula requires: - setup - test: @@ -209,7 +209,7 @@ workflows: - setup matrix: parameters: - golang_version: ["1.14.12", "1.15.5"] + golang_version: ["1.14.15", "1.15.8"] - test: name: << matrix.test_make_target >> resource_class: xlarge @@ -221,22 +221,37 @@ workflows: test_make_target: ["test-race", "test-txstore-rbf_bolt"] - test: name: test-shardwidth-22 + context: molecula shard_width: "22" resource_class: large requires: - setup - cluster-tests: + context: molecula requires: - setup - docker-build: context: molecula requires: - setup - - prerelease: + - release: + context: molecula requires: - - linter - - check-license-headers - - test-golang-1.15.5 + - setup + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ + - publish_release: + context: molecula + requires: + - release + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ - dockerhub-upload-unstable: context: molecula requires: diff --git a/.dockerignore b/.dockerignore index d5de893f3..0e869cde0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ -lattice +lattice/.git +lattice/node_modules +lattice/build +statik/statik.go +build diff --git a/Dockerfile b/Dockerfile index 955f499ed..20170e420 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,46 @@ -FROM golang:1.14.10 as builder +ARG GO_VERSION=latest -ARG BUILD_FLAGS +####################### +### Lattice builder ### +####################### + +FROM moleculacorp/nodejs:latest as lattice-builder +WORKDIR /lattice + +COPY lattice/package.json ./ +COPY lattice/yarn.lock ./ +RUN yarn install + +COPY lattice ./ +RUN yarn build + +###################### +### Pilosa builder ### +###################### + +FROM golang:${GO_VERSION} as pilosa-builder ARG MAKE_FLAGS +WORKDIR /pilosa -COPY . pilosa +RUN go get github.com/rakyll/statik -RUN cd pilosa && make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS} +COPY . ./ +COPY --from=lattice-builder /lattice/build /lattice +RUN /go/bin/statik -src=/lattice -dest=/pilosa -FROM alpine:3.12.1 +RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS} -LABEL maintainer "dev@pilosa.com" +##################### +### Pilosa runner ### +##################### + +FROM alpine:3.13.2 as runner + +LABEL maintainer "dev@molecula.com" RUN apk add --no-cache curl jq -COPY --from=builder /go/bin/pilosa /pilosa +COPY --from=pilosa-builder /pilosa/build/pilosa / COPY LICENSE /LICENSE COPY NOTICE /NOTICE diff --git a/Makefile b/Makefile index 84a2be13f..f93ea4f52 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,22 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf lattice CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) VARIANT = Molecula -VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) +GO=go +GOOS=$(shell $(GO) env GOOS) +GOARCH=$(shell $(GO) env GOARCH) +VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT) -X github.com/pilosa/pilosa/v2.TrialDeadline=$(TRIAL_DEADLINE)" -TRIAL_STRING = $(if $(TRIAL_DEADLINE),"-trial-$(TRIAL_DEADLINE)","") -GO_VERSION=1.14.10 -RELEASE ?= 0 -RELEASE_ENABLED = $(subst 0,,$(RELEASE)) -BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +GO_VERSION=1.15.8 +DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia define LICENSE_HASH_CODE @@ -43,15 +43,15 @@ clean: # Set up vendor directory using `go mod vendor` vendor: go.mod - go mod vendor + $(GO) mod vendor # Run test suite test: - go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v + $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v # Run test suite with race flag test-race: - CGO_ENABLED=1 go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v + CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v testv: topt testvsub @@ -66,7 +66,7 @@ testvsub: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ + $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i"; \ cd ..; \ done @@ -75,7 +75,7 @@ testvsub-race: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ - CGO_ENABLED=1 go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ + CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i -race"; \ cd ..; \ done @@ -84,7 +84,7 @@ tour: ./tournament.sh bench: - go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) + $(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) # Run test suite with coverage enabled cover: @@ -93,18 +93,18 @@ cover: # Run test suite with coverage enabled and view coverage results in browser cover-viz: cover - go tool cover -html=build/coverage.out + $(GO) tool cover -html=build/coverage.out # Compile Pilosa build: - go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa # Create a single release build under the build directory release-build: - $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa$(TRIAL_STRING)-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE README.md LICENSE build/pilosa$(TRIAL_STRING)-$(VERSION_ID) - tar -cvz -C build -f build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz pilosa$(TRIAL_STRING)-$(VERSION_ID)/ - @echo Created release build: build/pilosa$(TRIAL_STRING)-$(VERSION_ID).tar.gz + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp NOTICE README.md LICENSE build/pilosa$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz # Error out if there are untracked changes in Git check-clean: @@ -112,16 +112,16 @@ ifndef SKIP_CHECK_CLEAN $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) endif -# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. -release: check-clean generate-statik +# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release` +release: check-clean generate-statik-docker $(MAKE) release-build GOOS=darwin GOARCH=amd64 - $(MAKE) release-build GOOS=linux GOARCH=amd64 $(if $(IS_MACOS),DOCKER_BUILD=1) + $(MAKE) release-build GOOS=linux GOARCH=amd64 # Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI. release-sans-ui: check-clean rm -f statik/statik.go $(MAKE) release-build GOOS=darwin GOARCH=amd64 - $(MAKE) release-build GOOS=linux GOARCH=amd64 $(if $(IS_MACOS),DOCKER_BUILD=1) + $(MAKE) release-build GOOS=linux GOARCH=amd64 # try (e.g.) internal/clustertests/docker-compose-replication2.yml DOCKER_COMPOSE=internal/clustertests/docker-compose.yml @@ -141,28 +141,21 @@ clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down -v docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build -# Create prerelease builds -prerelease: - $(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\) - $(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release) - -prerelease-upload: - aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read - # Install Pilosa install: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa install-bench: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench # Ensure lattice is cloned and the pinned version is checked out lattice: git submodule update --init # Build the lattice assets -build-lattice: lattice require-yarn - cd lattice && yarn install && yarn build +build-lattice: lattice + docker build -t lattice:build ./lattice + export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE # Upgrade lattice to the latest version upgrade-lattice: lattice @@ -170,15 +163,19 @@ upgrade-lattice: lattice # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast - go generate github.com/pilosa/pilosa/v2/internal + $(GO) generate github.com/pilosa/pilosa/v2/internal # `go generate` statik assets (lattice UI) generate-statik: build-lattice require-statik - go generate github.com/pilosa/pilosa/v2/statik + $(GO) generate github.com/pilosa/pilosa/v2/statik + +# `go generate` statik assets (lattice UI) in Docker +generate-statik-docker: build-lattice + docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f" # `go generate` stringers generate-stringer: - go generate github.com/pilosa/pilosa/v2 + $(GO) generate github.com/pilosa/pilosa/v2 generate-pql: require-peg cd pql && peg -inline pql.peg && cd .. @@ -195,28 +192,49 @@ generate-proto-grpc: require-protoc require-protoc-gen-go # `go generate` all needed packages generate: generate-protoc generate-statik generate-stringer generate-pql +# Create release using Docker +docker-release: + $(MAKE) docker-build GOOS=linux GOARCH=amd64 + $(MAKE) docker-build GOOS=darwin GOARCH=amd64 + +# Build a release in Docker +docker-build: vendor lattice + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \ + --target pilosa-builder \ + --tag pilosa:build . + docker create --name pilosa-build pilosa:build + mkdir -p build/pilosa-$(VERSION_ID) + docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID) + cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID) + docker rm pilosa-build + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + # Create Docker image from Dockerfile -docker: vendor - docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . +docker-image: vendor lattice + docker build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \ + --tag pilosa:$(VERSION) . @echo Created docker image: pilosa:$(VERSION) +# Create docker image (alias) +docker: docker-image # alias + # Tag and push a Docker image docker-tag-push: vendor docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET) docker push $(DOCKER_TARGET) @echo Pushed docker image: $(DOCKER_TARGET) -# Compile Pilosa inside Docker container -docker-build: vendor - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) make build FLAGS="$(FLAGS) -mod=vendor" RELEASE=$(RELEASE) - # Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory. pilosa-keydump: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump # Install diagnostic pilosa-chk tool for string translations and fragment checksums. pilosa-chk: - go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk + $(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk pilosa-fsck: cd ./cmd/pilosa-fsck && make install && make release @@ -230,13 +248,13 @@ docker-test: # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: mv log.topt.roar log.topt.roar.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; go test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar + $(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l topt-race: mv log.topt.race log.topt.race.prev || true - $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 go test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race + $(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l @@ -295,27 +313,27 @@ install-statik: go get -u github.com/rakyll/statik install-stringer: - GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer + GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer install-protoc-gen-gofast: - GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast + GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast install-protoc-gen-go: - GO111MODULE=off go get -u github.com/golang/protobuf/protoc-gen-go + GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html install-peg: - GO111MODULE=off go get github.com/pointlander/peg + GO111MODULE=off $(GO) get github.com/pointlander/peg install-golangci-lint: - GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint install-gometalinter: - GO111MODULE=off go get -u github.com/alecthomas/gometalinter + GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install - GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode + GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode test-txstore-rbf: PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race diff --git a/api.go b/api.go index f0935a2f8..5815ce21b 100644 --- a/api.go +++ b/api.go @@ -17,6 +17,7 @@ package pilosa import ( + "bytes" "context" "encoding/binary" "encoding/csv" @@ -406,22 +407,33 @@ func importWorker(importWork chan importJob) { fallthrough case RequestActionSet: fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) - if err != nil { - return errors.Wrap(err, "importing pilosa roaring") - } - } else { - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) + data := viewData + if fileMagic != roaring.MagicNumber { + // if the view data arrives is in the "standard" roaring format, we must + // make a copy of data in order allow for the conversion to the pilosa roaring run format + // in field.importRoaring + data = make([]byte, len(viewData)) copy(data, viewData) - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + } + if j.req.UpdateExistence { + if ef := j.field.idx.existenceField(); ef != nil { + existence, err := combineForExistence(data) + if err != nil { + return errors.Wrap(err, "merging existence on roaring import") + } - if err != nil { - return errors.Wrap(err, "importing standard roaring") + err = ef.importRoaring(j.ctx, tx, existence, j.shard, "standard", false) + if err != nil { + return errors.Wrap(err, "updating existence on roaring import") + } } } + + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } return nil }(); err != nil { @@ -438,6 +450,24 @@ func importWorker(importWork chan importJob) { } } +// combineForExistence unions all rows in the fragment to be imported into a single row to update the existence field. TODO: It would probably be more efficient to only unmarshal the input data once, and use the calculated existence Bitmap directly rather than returning it to bytes, but most of our ingest paths update existence separately, so it's more important that this just be obviously correct at the moment. +func combineForExistence(inputRoaringData []byte) ([]byte, error) { + rowSize := uint64(1 << shardVsContainerExponent) + rit, err := roaring.NewRoaringIterator(inputRoaringData) + if err != nil { + return nil, err + } + bm := roaring.NewBitmap() + err = bm.MergeRoaringRawIteratorIntoExists(rit, rowSize) + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + + _, err = bm.WriteTo(buf) + return buf.Bytes(), err +} + // ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 121b632ca..9755d5bb6 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -478,12 +478,13 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - IndexCreatedAt: m.IndexCreatedAt, - FieldCreatedAt: m.FieldCreatedAt, - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, + UpdateExistence: m.UpdateExistence, } } @@ -1256,6 +1257,7 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Views = views m.IndexCreatedAt = pb.IndexCreatedAt m.FieldCreatedAt = pb.FieldCreatedAt + m.UpdateExistence = pb.UpdateExistence } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { diff --git a/executor.go b/executor.go index dcd6ccd12..0e869ff64 100644 --- a/executor.go +++ b/executor.go @@ -3005,10 +3005,12 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } } + ignoreLimit := sorter != nil // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) { - return e.executeGroupByShard(ctx, qcx, index, c, filter, shard, childRows, bases) + return e.executeGroupByShard(ctx, qcx, index, c, filter, shard, childRows, bases, ignoreLimit) } + // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other := findGroupCounts(prev) @@ -3563,7 +3565,7 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64, ignoreLimit bool) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard") defer span.Finish() @@ -3593,7 +3595,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri limit := int(^uint(0) >> 1) if lim, hasLimit, err := c.UintArg("limit"); err != nil { return nil, err - } else if hasLimit { + } else if !ignoreLimit && hasLimit { limit = int(lim) } diff --git a/executor_test.go b/executor_test.go index 63b015816..7e79baa14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -7188,6 +7188,17 @@ func variousQueries(t *testing.T, c *test.Cluster) { {"icecream", "userF"}, }) + // Create and populate "dinner" field. + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, "users", "dinner", [][2]string{ + {"leftovers", "userB"}, + {"pizza", "userA"}, + {"pizza", "userB"}, + {"chinese", "userA"}, + {"chinese", "userB"}, + {"chinese", "userF"}, + }) + // Create and populate "places_visited" time field. c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 @@ -7563,6 +7574,26 @@ pangolin,1,100 0,1,1 5,1,1 10,1,1 +`, + }, + { + query: "GroupBy(Rows(field=dinner), sort=\"count desc\", limit=2)", + csvVerifier: `chinese,3 +pizza,2 +`, + }, + { + query: "TopK(dinner)", + csvVerifier: `chinese,3 +pizza,2 +leftovers,1 +`, + }, + { + query: "TopK(field=dinner)", + csvVerifier: `chinese,3 +pizza,2 +leftovers,1 `, }, } diff --git a/handler.go b/handler.go index bbb1daa60..ade7c9c26 100644 --- a/handler.go +++ b/handler.go @@ -243,12 +243,13 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - IndexCreatedAt int64 - FieldCreatedAt int64 - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte + UpdateExistence bool } // ValidateWithTimestamp ensures that the payload of the request is valid. diff --git a/http/client_test.go b/http/client_test.go index 4a1beadf0..a10f524fa 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -1438,3 +1438,53 @@ func TestClient_ServerInfoHasBackend(t *testing.T) { } pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid } +func TestClient_ImportRoaringExists(t *testing.T) { + cluster := test.MustNewCluster(t, 1) + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + node := cluster.GetNode(0) + _, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + // Send import request. + host := node.URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + qr, err := node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf(" %v ", err) + } + got := qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, []uint64{}) { + t.Fatalf(" Row unexpected columns: got %+v expected: %+v", got, []uint64{}) + } + roaringReq.UpdateExistence = true + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + t.Fatal(err) + } + + expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537} + qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"}) + if err != nil { + t.Fatalf("Query error: %+v", err) + } + got = qr.Results[0].(*pilosa.Row).Columns() + if !reflect.DeepEqual(got, expected) { + t.Fatalf("All unexpected columns: got %+v expected: %+v", got, expected) + } + +} diff --git a/internal/public.pb.go b/internal/public.pb.go index 5f0c5277d..98fc6ef34 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2495,6 +2495,7 @@ type ImportRoaringRequest struct { Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` + UpdateExistence bool `protobuf:"varint,7,opt,name=UpdateExistence,proto3" json:"UpdateExistence,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2575,6 +2576,13 @@ func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { return 0 } +func (m *ImportRoaringRequest) GetUpdateExistence() bool { + if m != nil { + return m.UpdateExistence + } + return false +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` @@ -2761,117 +2769,119 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1757 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe4, 0x48, - 0x15, 0x8f, 0xdb, 0xee, 0x7f, 0xaf, 0x3b, 0x99, 0x6c, 0x4d, 0xcf, 0x62, 0x0d, 0x99, 0x6c, 0x63, - 0x05, 0xb6, 0x41, 0xab, 0xac, 0x12, 0x76, 0x60, 0x0e, 0xfc, 0xd9, 0x64, 0x3a, 0x4b, 0xac, 0x21, - 0xd9, 0xa1, 0x12, 0x65, 0xc5, 0x05, 0xc9, 0xe9, 0x2e, 0x7a, 0x2d, 0xdc, 0xed, 0xc6, 0xed, 0xde, - 0x4e, 0x2e, 0x48, 0x7c, 0x86, 0xbd, 0x70, 0x43, 0xdc, 0xf8, 0x1c, 0x5c, 0xe0, 0xc8, 0x11, 0x89, - 0x0b, 0x1a, 0xb8, 0xf2, 0x1d, 0xd0, 0x7b, 0xe5, 0x72, 0x95, 0xdd, 0x4e, 0x26, 0x1a, 0x71, 0xab, - 0xf7, 0xa7, 0x5e, 0xd5, 0xfb, 0xbd, 0x57, 0xef, 0x3d, 0x1b, 0xba, 0xf3, 0xe5, 0x75, 0x14, 0x8e, - 0xf6, 0xe7, 0x49, 0x9c, 0xc6, 0xac, 0x15, 0xce, 0x52, 0x91, 0xcc, 0x82, 0xc8, 0xfb, 0xa3, 0x05, - 0x36, 0x8f, 0x57, 0xcc, 0x85, 0xe6, 0xcb, 0x38, 0x5a, 0x4e, 0x67, 0x0b, 0xd7, 0xea, 0xdb, 0x03, - 0x87, 0x2b, 0x92, 0x31, 0x70, 0x5e, 0x89, 0xdb, 0x85, 0x6b, 0xf7, 0xed, 0x41, 0x9b, 0xd3, 0x9a, - 0xed, 0x41, 0xfd, 0x28, 0x4d, 0x93, 0x85, 0x5b, 0xeb, 0xdb, 0x83, 0xce, 0xe1, 0xd6, 0xbe, 0xb2, - 0xb7, 0x8f, 0x6c, 0x2e, 0x85, 0x68, 0x93, 0xc7, 0x41, 0x12, 0xce, 0x26, 0xae, 0xd3, 0xb7, 0x06, - 0x5d, 0xae, 0x48, 0xd6, 0x83, 0xba, 0x3f, 0x1b, 0x8b, 0x1b, 0xb7, 0xde, 0xb7, 0x06, 0x6d, 0x2e, - 0x09, 0xe4, 0x7e, 0x16, 0x8a, 0x68, 0xec, 0x36, 0x24, 0x97, 0x08, 0x6f, 0x1f, 0xda, 0x3c, 0x5e, - 0x9d, 0x05, 0x69, 0x12, 0xde, 0xb0, 0x6f, 0x81, 0xc3, 0xe3, 0x95, 0xbc, 0x63, 0xe7, 0x70, 0x53, - 0x9f, 0xcb, 0xe3, 0x15, 0x27, 0x91, 0x77, 0x06, 0xed, 0x8b, 0x70, 0x32, 0x13, 0x63, 0x74, 0xeb, - 0x03, 0xb0, 0x5f, 0xc7, 0xa8, 0x6e, 0xad, 0xab, 0xa3, 0x04, 0x15, 0xce, 0xc5, 0xc4, 0xad, 0x55, - 0x2a, 0x9c, 0x8b, 0x89, 0xf7, 0x02, 0xb6, 0x78, 0xbc, 0xf2, 0xc7, 0x62, 0x96, 0x86, 0xbf, 0x0e, - 0x45, 0x42, 0x80, 0xe4, 0x77, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x14, 0x1a, - 0xfe, 0xf0, 0xe7, 0xe1, 0x22, 0x65, 0xdb, 0x60, 0xfb, 0x43, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0xef, - 0x9d, 0xdc, 0xa4, 0x49, 0x30, 0x4a, 0xc5, 0xd8, 0x1f, 0x4a, 0xa8, 0xd9, 0x16, 0xd4, 0xfc, 0x21, - 0xdd, 0xd5, 0xe1, 0x35, 0x7f, 0xc8, 0xf6, 0xc0, 0xb9, 0x0a, 0x22, 0x05, 0xf2, 0xb6, 0xbe, 0x9c, - 0x34, 0xcb, 0x49, 0xea, 0x5d, 0x17, 0x4c, 0x65, 0x38, 0xbd, 0x0f, 0x0d, 0x42, 0x4f, 0x1e, 0xda, - 0xe6, 0x19, 0xc5, 0x9e, 0xeb, 0x30, 0x4b, 0xab, 0xdf, 0xd4, 0x56, 0xd7, 0x2e, 0x94, 0xe7, 0x80, - 0xf7, 0x0c, 0x9a, 0xaf, 0xc4, 0x2d, 0xf9, 0xa2, 0x3c, 0xb5, 0x0c, 0x4f, 0xff, 0x69, 0xc1, 0xe3, - 0x7c, 0xf7, 0x65, 0x70, 0x1d, 0x89, 0xab, 0x20, 0x5a, 0x0a, 0xb6, 0xa7, 0xfc, 0xb6, 0xaa, 0xee, - 0x7f, 0xba, 0x41, 0x58, 0xb0, 0x0f, 0x73, 0xec, 0x50, 0xed, 0x3d, 0xad, 0x96, 0x1d, 0x79, 0xba, - 0x91, 0x65, 0xdd, 0x0e, 0xb4, 0x8e, 0x2f, 0x7c, 0x32, 0xed, 0xda, 0x7d, 0x6b, 0x60, 0x9f, 0x6e, - 0xf0, 0x9c, 0xc3, 0x9e, 0x42, 0xf3, 0x6c, 0x99, 0x8a, 0x1b, 0x7f, 0x48, 0xd9, 0xe6, 0x9c, 0x6e, - 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x25, 0x6e, 0x65, 0xca, 0xe1, 0x4e, 0xc5, 0x61, 0x3d, 0x70, - 0x8e, 0xe3, 0x38, 0xa2, 0xb4, 0x6b, 0xe1, 0x69, 0x48, 0x1d, 0x37, 0xa1, 0x4e, 0x86, 0xbd, 0xdf, - 0x41, 0xaf, 0xe8, 0x5c, 0x16, 0x2e, 0x06, 0x36, 0xda, 0xb3, 0x32, 0x7b, 0x48, 0xb0, 0x6d, 0x0a, - 0x61, 0x2d, 0x3b, 0x1f, 0x83, 0xf8, 0x1c, 0x1a, 0x64, 0x46, 0x3e, 0xa0, 0xce, 0xe1, 0xb3, 0x0a, - 0xc0, 0x35, 0x64, 0x3c, 0x53, 0x3e, 0x6e, 0x13, 0xe2, 0x9f, 0x27, 0xfe, 0xd0, 0xfb, 0x71, 0x19, - 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x3c, 0x98, 0x0a, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0x2e, 0x6f, 0xe7, - 0x82, 0x2e, 0xd0, 0xe6, 0xb4, 0xf6, 0x7e, 0x6f, 0xc1, 0x56, 0x71, 0x3f, 0xde, 0xc9, 0xc8, 0x8e, - 0x7b, 0xee, 0x44, 0x5a, 0x79, 0xf2, 0xbc, 0x28, 0x27, 0xcf, 0xee, 0x5d, 0xfb, 0xca, 0xf9, 0xf3, - 0x13, 0x70, 0x5e, 0x07, 0x61, 0xb2, 0x96, 0xe1, 0xdb, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x58, - 0xd4, 0x5f, 0xc6, 0xcb, 0x59, 0x2a, 0x31, 0xe4, 0x92, 0xf0, 0x4e, 0xa0, 0x8d, 0xfb, 0xa5, 0xe3, - 0x9e, 0x34, 0x96, 0xa5, 0x95, 0x51, 0x7b, 0x90, 0xcb, 0xe5, 0x41, 0x79, 0x29, 0xa9, 0x99, 0xa5, - 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x69, 0x67, 0x0f, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, - 0x1d, 0x96, 0x9e, 0x61, 0x01, 0x4b, 0x7f, 0xf0, 0x09, 0x8a, 0x65, 0x42, 0xe2, 0x6d, 0x6c, 0x9e, - 0xa5, 0xcc, 0x12, 0x5a, 0x12, 0xba, 0x78, 0xa5, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x59, 0x19, - 0x2a, 0x3f, 0x89, 0xc0, 0x67, 0xcb, 0xe3, 0x95, 0x86, 0x24, 0xa3, 0xd8, 0xb7, 0xd5, 0x29, 0x0e, - 0xf9, 0xfc, 0xc8, 0x78, 0x4a, 0x78, 0x0b, 0x75, 0xec, 0xaf, 0x00, 0x7e, 0x96, 0xc4, 0xcb, 0x39, - 0x81, 0xc6, 0x06, 0x50, 0x27, 0x2a, 0xf3, 0x8f, 0xe9, 0x4d, 0xea, 0x6e, 0x5c, 0x2a, 0x54, 0x83, - 0x8e, 0xc1, 0x39, 0x9a, 0x4c, 0xe4, 0x4b, 0xe3, 0xb8, 0xc4, 0x54, 0x6a, 0x5d, 0x05, 0x51, 0x2e, - 0xbe, 0x0a, 0xa2, 0xcc, 0x6f, 0x5c, 0x16, 0xcd, 0xd8, 0xca, 0xcc, 0x53, 0x68, 0x7d, 0x16, 0xc5, - 0x41, 0x8a, 0xca, 0x68, 0xcb, 0xe2, 0x39, 0xcd, 0x0e, 0x00, 0x86, 0x62, 0x14, 0x4e, 0x83, 0x08, - 0xa5, 0x4e, 0xb9, 0x00, 0x64, 0x32, 0x6e, 0x28, 0x79, 0xcf, 0xa1, 0x99, 0x51, 0xd5, 0xd8, 0x23, - 0xf7, 0x62, 0x14, 0x44, 0x42, 0xdd, 0x82, 0x08, 0xef, 0x0b, 0xd8, 0x94, 0xc9, 0x88, 0xad, 0xe9, - 0x42, 0xa4, 0x0f, 0x48, 0xc5, 0x07, 0x35, 0x39, 0xef, 0xcf, 0x16, 0x38, 0xb8, 0x52, 0x06, 0x2c, - 0x6d, 0xc0, 0x7c, 0x8d, 0x8e, 0x7c, 0x8d, 0xac, 0x0f, 0x9d, 0x8b, 0x14, 0x7b, 0xa0, 0x2e, 0x63, - 0x6d, 0x6e, 0xb2, 0x10, 0x2f, 0x7f, 0x96, 0xea, 0x70, 0xdb, 0x3c, 0xa7, 0xd9, 0x0e, 0xb4, 0xb1, - 0x36, 0x49, 0x21, 0x16, 0xb2, 0x16, 0xd7, 0x0c, 0xb6, 0x0b, 0xa0, 0x90, 0x5d, 0x0a, 0xaa, 0x66, - 0x16, 0x37, 0x38, 0xde, 0xc7, 0xd0, 0xc4, 0x9b, 0x9e, 0x05, 0x73, 0xed, 0x9b, 0x75, 0x9f, 0x6f, - 0x7f, 0xaa, 0x41, 0xf7, 0x17, 0x4b, 0x91, 0xdc, 0x72, 0xf1, 0xdb, 0xa5, 0x58, 0xa4, 0x88, 0x2d, - 0xd1, 0x2a, 0x97, 0x89, 0xc0, 0xac, 0xbd, 0xf8, 0x32, 0x48, 0xc6, 0x12, 0x29, 0x87, 0x67, 0x14, - 0xfa, 0xaa, 0x31, 0x5f, 0x90, 0xaf, 0x2d, 0x6e, 0xb2, 0x28, 0xdf, 0xc5, 0x34, 0x4e, 0x95, 0x33, - 0x19, 0xc5, 0x06, 0xf0, 0xe8, 0xe4, 0x66, 0x14, 0x2d, 0xc7, 0x82, 0xc7, 0x2b, 0xb9, 0x9b, 0x8a, - 0x33, 0x2f, 0xb3, 0xd9, 0x77, 0xb0, 0xb8, 0x11, 0x4b, 0x95, 0xa6, 0x26, 0x29, 0x96, 0xb8, 0xec, - 0x00, 0xba, 0x27, 0xd3, 0x6b, 0x31, 0x1e, 0x8b, 0xf1, 0x30, 0x48, 0x03, 0xb7, 0x55, 0x35, 0x40, - 0x14, 0x54, 0xd8, 0x1e, 0x6c, 0xbe, 0x4e, 0xc4, 0x65, 0x12, 0xcc, 0x16, 0x51, 0x90, 0x8a, 0xb1, - 0xdb, 0x26, 0xcb, 0x45, 0xa6, 0xf7, 0xb5, 0x05, 0x9b, 0x19, 0x46, 0x8b, 0x79, 0x3c, 0x5b, 0x08, - 0x4c, 0x84, 0x93, 0x24, 0x51, 0x89, 0x70, 0x92, 0x24, 0xec, 0x63, 0x68, 0x72, 0xb1, 0x58, 0x46, - 0xa9, 0xca, 0xa5, 0x27, 0xfa, 0x5c, 0xb5, 0x77, 0x19, 0xa5, 0x5c, 0x69, 0xb1, 0x9f, 0xc2, 0x56, - 0x21, 0x5b, 0x55, 0xf3, 0xf8, 0x86, 0xde, 0x57, 0x90, 0xf3, 0x92, 0xba, 0xf7, 0xdf, 0x3a, 0x74, - 0x0c, 0xcb, 0x79, 0x2a, 0x22, 0x8a, 0x9b, 0x59, 0x2a, 0x7e, 0x40, 0x93, 0xdf, 0x1d, 0xb3, 0x11, - 0x56, 0xae, 0x2e, 0x58, 0xe7, 0x59, 0xf2, 0x5a, 0xe7, 0xba, 0x5c, 0xda, 0xf7, 0x95, 0x4b, 0x9c, - 0x23, 0xbf, 0x0c, 0x66, 0x13, 0x31, 0xa6, 0xe4, 0x6d, 0x71, 0x45, 0xb2, 0x7d, 0x5d, 0x3b, 0x28, - 0xda, 0x85, 0x8a, 0xa4, 0x24, 0x5c, 0xd7, 0x17, 0x59, 0x0b, 0x71, 0x7e, 0x68, 0xca, 0xac, 0x92, - 0x14, 0xfb, 0x11, 0x6c, 0x7d, 0x1e, 0x8d, 0x75, 0x9d, 0x5b, 0x64, 0xb1, 0xec, 0x69, 0x6b, 0x5a, - 0xc8, 0x4b, 0xba, 0xec, 0xd3, 0xf2, 0x38, 0x47, 0x51, 0xed, 0x1c, 0xba, 0x05, 0xff, 0x0d, 0x39, - 0x2f, 0x8f, 0x7f, 0x07, 0xc6, 0x7c, 0xe9, 0x02, 0x6d, 0x7e, 0xac, 0x37, 0xe7, 0x22, 0x6e, 0x4c, - 0xa1, 0x9f, 0x98, 0x7d, 0xc7, 0xed, 0xd0, 0x9e, 0x5e, 0x11, 0x3f, 0x29, 0xe3, 0x66, 0x7f, 0x3a, - 0x30, 0x9a, 0x9e, 0xdb, 0x2d, 0x1f, 0x94, 0x8b, 0xb8, 0xd1, 0x1a, 0xfd, 0x8a, 0x59, 0xd0, 0xdd, - 0xa4, 0xad, 0xd5, 0x83, 0x9e, 0x54, 0xe1, 0x15, 0x13, 0xe4, 0xa7, 0xe5, 0xa9, 0xc1, 0xdd, 0x2a, - 0x03, 0x55, 0x94, 0xf3, 0xf2, 0x94, 0x71, 0x60, 0x0c, 0xee, 0xee, 0xa3, 0xf2, 0xfd, 0x73, 0x11, - 0x37, 0xc6, 0xfb, 0x1f, 0x42, 0xc7, 0x0c, 0xec, 0x36, 0x6d, 0x7a, 0x52, 0x15, 0xd8, 0x05, 0x37, - 0x35, 0xbd, 0xbf, 0xd6, 0x60, 0xd3, 0x9f, 0xce, 0xe3, 0x24, 0x35, 0x4a, 0x95, 0xfc, 0xc4, 0xb0, - 0x2a, 0x3f, 0x31, 0x6a, 0xa5, 0x66, 0x4c, 0x25, 0x8b, 0x4a, 0x94, 0xc3, 0x25, 0x61, 0x24, 0xa0, - 0x53, 0x48, 0xc0, 0x1d, 0x68, 0xcb, 0xd7, 0x86, 0xa2, 0x3a, 0x89, 0x34, 0x43, 0x7e, 0xf4, 0xac, - 0x68, 0xa0, 0x6d, 0xd2, 0x88, 0xac, 0x48, 0x2c, 0xcf, 0x52, 0x8d, 0x84, 0x2d, 0x12, 0x1a, 0x1c, - 0x94, 0x5f, 0x86, 0x53, 0xb1, 0x48, 0x83, 0xe9, 0x1c, 0xeb, 0x9d, 0x3d, 0xb0, 0xb9, 0xc1, 0xc1, - 0x52, 0x47, 0x4e, 0xbc, 0x4c, 0x04, 0x56, 0x9e, 0xa3, 0x94, 0x52, 0xd7, 0xe6, 0x25, 0x2e, 0xea, - 0x91, 0x5b, 0x5a, 0x0f, 0xa4, 0x5e, 0x91, 0x4b, 0xed, 0x3a, 0x12, 0x41, 0x42, 0x09, 0xd9, 0xe2, - 0x92, 0xf0, 0xfe, 0x51, 0x03, 0x26, 0x91, 0x94, 0x03, 0xe9, 0xff, 0x0d, 0xce, 0xfb, 0x61, 0x2b, - 0x82, 0xd3, 0x5c, 0x03, 0xe7, 0xfd, 0x7c, 0x8c, 0x96, 0xc0, 0x64, 0x14, 0xf6, 0x18, 0xdd, 0xe1, - 0x24, 0xaa, 0x16, 0x37, 0x59, 0xcc, 0x83, 0xae, 0xd1, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, 0x02, 0xaf, - 0x02, 0x5a, 0x78, 0x20, 0xb4, 0x9d, 0xfb, 0xa1, 0xed, 0x9a, 0xd0, 0x7e, 0x6d, 0x41, 0xf7, 0x28, - 0x8d, 0xa7, 0xe1, 0x88, 0x8b, 0x51, 0x9c, 0x8c, 0xef, 0x06, 0x55, 0xc2, 0x57, 0x33, 0xe1, 0xdb, - 0x07, 0xdb, 0xff, 0x2a, 0xc9, 0x8a, 0xef, 0x8e, 0x31, 0x00, 0xae, 0xc5, 0x8a, 0xa3, 0x22, 0xfb, - 0x10, 0x6a, 0x7e, 0x42, 0x99, 0x5b, 0x68, 0x1b, 0x85, 0x47, 0xc2, 0x6b, 0x7e, 0xe2, 0x7d, 0x04, - 0x3d, 0x79, 0x29, 0x25, 0xca, 0xda, 0x58, 0x0f, 0xea, 0x27, 0x49, 0x12, 0xab, 0x46, 0x26, 0x09, - 0xef, 0x06, 0x7a, 0x79, 0xf3, 0xc3, 0xc0, 0xbc, 0x4b, 0x7e, 0x54, 0xfd, 0x51, 0xe8, 0x43, 0xe7, - 0x3c, 0x4e, 0xbf, 0x48, 0xc2, 0x94, 0x6a, 0x8d, 0xec, 0x1d, 0x26, 0xcb, 0xfb, 0x2e, 0x3c, 0x29, - 0x9d, 0xac, 0xfb, 0x2d, 0xa6, 0x94, 0xad, 0xbf, 0xae, 0x2f, 0xe0, 0x71, 0xae, 0xea, 0x0f, 0xdf, - 0xe9, 0x8e, 0xeb, 0x46, 0xbf, 0x67, 0x78, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xbc, 0xf1, 0x8e, 0xc1, - 0xcd, 0xd0, 0x94, 0x3f, 0x3c, 0xb2, 0x1b, 0x5c, 0x85, 0x62, 0x75, 0xd7, 0x77, 0x1b, 0x4d, 0x25, - 0x35, 0xfa, 0x4d, 0x42, 0x6b, 0xef, 0x3f, 0x16, 0xf4, 0xaa, 0x8c, 0xe8, 0xe4, 0xb2, 0x8c, 0xe4, - 0x62, 0x2f, 0xa0, 0xfe, 0x55, 0x28, 0x56, 0x6a, 0xc2, 0xf0, 0xd6, 0x42, 0xbe, 0x76, 0x13, 0x2e, - 0x37, 0xe0, 0xd3, 0x3a, 0x1a, 0xa5, 0x61, 0x3c, 0x53, 0x1f, 0x1d, 0x92, 0xc2, 0x73, 0x8e, 0xa3, - 0x78, 0xf4, 0x1b, 0xf9, 0x39, 0xcd, 0x25, 0x51, 0xf1, 0x54, 0xea, 0x0f, 0x7c, 0x2a, 0x8d, 0xaa, - 0xa7, 0xe2, 0xfd, 0xc5, 0x52, 0x58, 0x19, 0x83, 0xe1, 0x5b, 0x23, 0xa6, 0x1f, 0x88, 0xad, 0x1e, - 0x88, 0x2b, 0xa7, 0x5b, 0x3d, 0xc4, 0x2b, 0x12, 0x27, 0x6a, 0x5c, 0xd2, 0xbf, 0x14, 0x87, 0xa2, - 0x94, 0xd3, 0x6f, 0xa9, 0x4a, 0xeb, 0xce, 0x36, 0xaa, 0x9c, 0xf5, 0x7e, 0x59, 0xe8, 0x5b, 0x68, - 0xf4, 0x68, 0x32, 0x49, 0xc4, 0x24, 0x48, 0x55, 0x9c, 0x35, 0x83, 0x7d, 0x04, 0x0d, 0x52, 0x56, - 0xa1, 0xaa, 0x1e, 0x5c, 0x32, 0x9d, 0xe3, 0xed, 0xbf, 0xbd, 0xd9, 0xb5, 0xfe, 0xfe, 0x66, 0xd7, - 0xfa, 0xd7, 0x9b, 0x5d, 0xeb, 0x0f, 0xff, 0xde, 0xdd, 0xb8, 0x6e, 0xd0, 0x3f, 0xbc, 0xef, 0xff, - 0x2f, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x79, 0xf3, 0x2b, 0xd3, 0x13, 0x00, 0x00, + // 1779 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x49, + 0x15, 0x4f, 0xbb, 0xdb, 0xb1, 0xfd, 0xec, 0x64, 0xb2, 0x35, 0x9e, 0xa5, 0x35, 0xcc, 0x64, 0x43, + 0x2b, 0xb0, 0x06, 0xad, 0xb2, 0xca, 0xb0, 0x03, 0x73, 0xe0, 0xcf, 0x26, 0xe3, 0x2c, 0x69, 0x0d, + 0x93, 0x1d, 0x2a, 0x43, 0x56, 0x5c, 0x90, 0x3a, 0x76, 0xe1, 0x6d, 0xd1, 0x76, 0x9b, 0x76, 0x79, + 0x9d, 0x5c, 0x90, 0xf8, 0x02, 0x5c, 0xf6, 0xc2, 0x0d, 0x71, 0xe3, 0x73, 0x70, 0x81, 0x23, 0x47, + 0x24, 0x2e, 0x68, 0x38, 0xf3, 0x1d, 0xd0, 0x7b, 0xd5, 0xd5, 0x55, 0xdd, 0xee, 0x64, 0xa3, 0xd1, + 0xde, 0xea, 0xfd, 0xa9, 0x57, 0xf5, 0x7e, 0xef, 0xd5, 0x7b, 0xaf, 0x1b, 0x7a, 0xf3, 0xe5, 0x65, + 0x12, 0x8f, 0x0e, 0xe6, 0x59, 0x2a, 0x53, 0xd6, 0x8e, 0x67, 0x52, 0x64, 0xb3, 0x28, 0x09, 0xfe, + 0xec, 0x80, 0xcb, 0xd3, 0x15, 0xf3, 0xa1, 0xf5, 0x3c, 0x4d, 0x96, 0xd3, 0xd9, 0xc2, 0x77, 0xf6, + 0xdc, 0x81, 0xc7, 0x35, 0xc9, 0x18, 0x78, 0x2f, 0xc4, 0xf5, 0xc2, 0x77, 0xf7, 0xdc, 0x41, 0x87, + 0xd3, 0x9a, 0xed, 0x43, 0xf3, 0x48, 0xca, 0x6c, 0xe1, 0x37, 0xf6, 0xdc, 0x41, 0xf7, 0xc9, 0xf6, + 0x81, 0xb6, 0x77, 0x80, 0x6c, 0xae, 0x84, 0x68, 0x93, 0xa7, 0x51, 0x16, 0xcf, 0x26, 0xbe, 0xb7, + 0xe7, 0x0c, 0x7a, 0x5c, 0x93, 0xac, 0x0f, 0xcd, 0x70, 0x36, 0x16, 0x57, 0x7e, 0x73, 0xcf, 0x19, + 0x74, 0xb8, 0x22, 0x90, 0xfb, 0x49, 0x2c, 0x92, 0xb1, 0xbf, 0xa9, 0xb8, 0x44, 0x04, 0x07, 0xd0, + 0xe1, 0xe9, 0xea, 0x65, 0x24, 0xb3, 0xf8, 0x8a, 0x7d, 0x0b, 0x3c, 0x9e, 0xae, 0xd4, 0x1d, 0xbb, + 0x4f, 0xb6, 0xcc, 0xb9, 0x3c, 0x5d, 0x71, 0x12, 0x05, 0x2f, 0xa1, 0x73, 0x1e, 0x4f, 0x66, 0x62, + 0x8c, 0x6e, 0xbd, 0x07, 0xee, 0xab, 0x14, 0xd5, 0x9d, 0x75, 0x75, 0x94, 0xa0, 0xc2, 0x99, 0x98, + 0xf8, 0x8d, 0x5a, 0x85, 0x33, 0x31, 0x09, 0x9e, 0xc1, 0x36, 0x4f, 0x57, 0xe1, 0x58, 0xcc, 0x64, + 0xfc, 0x9b, 0x58, 0x64, 0x04, 0x48, 0x71, 0x07, 0x4f, 0x1d, 0x5a, 0x80, 0xd4, 0x30, 0x20, 0x05, + 0x0f, 0x61, 0x33, 0x1c, 0xfe, 0x3c, 0x5e, 0x48, 0xb6, 0x03, 0x6e, 0x38, 0xd4, 0x1b, 0x70, 0x19, + 0x84, 0xf0, 0xce, 0xc9, 0x95, 0xcc, 0xa2, 0x91, 0x14, 0xe3, 0x70, 0xa8, 0xa0, 0x66, 0xdb, 0xd0, + 0x08, 0x87, 0x74, 0x57, 0x8f, 0x37, 0xc2, 0x21, 0xdb, 0x07, 0xef, 0x22, 0x4a, 0x34, 0xc8, 0x3b, + 0xe6, 0x72, 0xca, 0x2c, 0x27, 0x69, 0x70, 0x59, 0x32, 0x95, 0xe3, 0xf4, 0x2e, 0x6c, 0x12, 0x7a, + 0xea, 0xd0, 0x0e, 0xcf, 0x29, 0xf6, 0xd4, 0x84, 0x59, 0x59, 0xfd, 0xa6, 0xb1, 0xba, 0x76, 0xa1, + 0x22, 0x07, 0x82, 0xc7, 0xd0, 0x7a, 0x21, 0xae, 0xc9, 0x17, 0xed, 0xa9, 0x63, 0x79, 0xfa, 0x6f, + 0x07, 0xee, 0x17, 0xbb, 0x5f, 0x47, 0x97, 0x89, 0xb8, 0x88, 0x92, 0xa5, 0x60, 0xfb, 0xda, 0x6f, + 0xa7, 0xee, 0xfe, 0xa7, 0x1b, 0x84, 0x05, 0x7b, 0xbf, 0xc0, 0x0e, 0xd5, 0xde, 0x31, 0x6a, 0xf9, + 0x91, 0xa7, 0x1b, 0x79, 0xd6, 0x3d, 0x82, 0xf6, 0xf1, 0x79, 0x48, 0xa6, 0x7d, 0x77, 0xcf, 0x19, + 0xb8, 0xa7, 0x1b, 0xbc, 0xe0, 0xb0, 0x87, 0xd0, 0x7a, 0xb9, 0x94, 0xe2, 0x2a, 0x1c, 0x52, 0xb6, + 0x79, 0xa7, 0x1b, 0x5c, 0x33, 0x70, 0x27, 0x2d, 0x5f, 0x88, 0x6b, 0x95, 0x72, 0xb8, 0x53, 0x73, + 0x58, 0x1f, 0xbc, 0xe3, 0x34, 0x4d, 0x28, 0xed, 0xda, 0x78, 0x1a, 0x52, 0xc7, 0x2d, 0x68, 0x92, + 0xe1, 0xe0, 0xf7, 0xd0, 0x2f, 0x3b, 0x97, 0x87, 0x8b, 0x81, 0x8b, 0xf6, 0x9c, 0xdc, 0x1e, 0x12, + 0x6c, 0x87, 0x42, 0xd8, 0xc8, 0xcf, 0xc7, 0x20, 0x3e, 0x85, 0x4d, 0x32, 0xa3, 0x1e, 0x50, 0xf7, + 0xc9, 0xe3, 0x1a, 0xc0, 0x0d, 0x64, 0x3c, 0x57, 0x3e, 0xee, 0x10, 0xe2, 0x9f, 0x66, 0xe1, 0x30, + 0xf8, 0x71, 0x15, 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x2c, 0x9a, 0x0a, 0x75, 0x3e, 0xa7, 0x35, 0xf2, + 0x5e, 0x5f, 0xcf, 0x05, 0x5d, 0xa0, 0xc3, 0x69, 0x1d, 0xfc, 0xc1, 0x81, 0xed, 0xf2, 0x7e, 0xbc, + 0x93, 0x95, 0x1d, 0xb7, 0xdc, 0x89, 0xb4, 0x8a, 0xe4, 0x79, 0x56, 0x4d, 0x9e, 0xdd, 0x9b, 0xf6, + 0x55, 0xf3, 0xe7, 0x27, 0xe0, 0xbd, 0x8a, 0xe2, 0x6c, 0x2d, 0xc3, 0x77, 0x14, 0x84, 0x2e, 0x5d, + 0xd7, 0x55, 0xb1, 0x68, 0x3e, 0x4f, 0x97, 0x33, 0xa9, 0x30, 0xe4, 0x8a, 0x08, 0x4e, 0xa0, 0x83, + 0xfb, 0x95, 0xe3, 0x81, 0x32, 0x96, 0xa7, 0x95, 0x55, 0x7b, 0x90, 0xcb, 0xd5, 0x41, 0x45, 0x29, + 0x69, 0xd8, 0xa5, 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x65, 0x67, 0x1f, 0x9a, 0x44, 0xe5, 0x20, 0x54, + 0x0d, 0x29, 0xe1, 0x0d, 0x96, 0x1e, 0x63, 0x01, 0x93, 0x3f, 0xf8, 0x08, 0xc5, 0x2a, 0x21, 0xf1, + 0x36, 0x2e, 0xcf, 0x53, 0x66, 0x09, 0x6d, 0x05, 0x5d, 0xba, 0x32, 0x06, 0x1c, 0xcb, 0x00, 0x72, + 0xb1, 0xac, 0x0c, 0xb5, 0x9f, 0x44, 0xe0, 0xb3, 0xe5, 0xe9, 0xca, 0x40, 0x92, 0x53, 0xec, 0xdb, + 0xfa, 0x14, 0x8f, 0x7c, 0xbe, 0x67, 0x3d, 0x25, 0xbc, 0x85, 0x3e, 0xf6, 0xd7, 0x00, 0x3f, 0xcb, + 0xd2, 0xe5, 0x9c, 0x40, 0x63, 0x03, 0x68, 0x12, 0x95, 0xfb, 0xc7, 0xcc, 0x26, 0x7d, 0x37, 0xae, + 0x14, 0xea, 0x41, 0xc7, 0xe0, 0x1c, 0x4d, 0x26, 0xea, 0xa5, 0x71, 0x5c, 0x62, 0x2a, 0xb5, 0x2f, + 0xa2, 0xa4, 0x10, 0x5f, 0x44, 0x49, 0xee, 0x37, 0x2e, 0xcb, 0x66, 0x5c, 0x6d, 0xe6, 0x21, 0xb4, + 0x3f, 0x49, 0xd2, 0x48, 0xa2, 0x32, 0xda, 0x72, 0x78, 0x41, 0xb3, 0x43, 0x80, 0xa1, 0x18, 0xc5, + 0xd3, 0x28, 0x41, 0xa9, 0x57, 0x2d, 0x00, 0xb9, 0x8c, 0x5b, 0x4a, 0xc1, 0x53, 0x68, 0xe5, 0x54, + 0x3d, 0xf6, 0xc8, 0x3d, 0x1f, 0x45, 0x89, 0xd0, 0xb7, 0x20, 0x22, 0xf8, 0x0c, 0xb6, 0x54, 0x32, + 0x62, 0x6b, 0x3a, 0x17, 0xf2, 0x0e, 0xa9, 0x78, 0xa7, 0x26, 0x17, 0xfc, 0xd5, 0x01, 0x0f, 0x57, + 0xda, 0x80, 0x63, 0x0c, 0xd8, 0xaf, 0xd1, 0x53, 0xaf, 0x91, 0xed, 0x41, 0xf7, 0x5c, 0x62, 0x0f, + 0x34, 0x65, 0xac, 0xc3, 0x6d, 0x16, 0xe2, 0x15, 0xce, 0xa4, 0x09, 0xb7, 0xcb, 0x0b, 0x9a, 0x3d, + 0x82, 0x0e, 0xd6, 0x26, 0x25, 0xc4, 0x42, 0xd6, 0xe6, 0x86, 0xc1, 0x76, 0x01, 0x34, 0xb2, 0x4b, + 0x41, 0xd5, 0xcc, 0xe1, 0x16, 0x27, 0xf8, 0x10, 0x5a, 0x78, 0xd3, 0x97, 0xd1, 0xdc, 0xf8, 0xe6, + 0xdc, 0xe6, 0xdb, 0x5f, 0x1a, 0xd0, 0xfb, 0xc5, 0x52, 0x64, 0xd7, 0x5c, 0xfc, 0x6e, 0x29, 0x16, + 0x12, 0xb1, 0x25, 0x5a, 0xe7, 0x32, 0x11, 0x98, 0xb5, 0xe7, 0x9f, 0x47, 0xd9, 0x58, 0x21, 0xe5, + 0xf1, 0x9c, 0x42, 0x5f, 0x0d, 0xe6, 0x0b, 0xf2, 0xb5, 0xcd, 0x6d, 0x16, 0xe5, 0xbb, 0x98, 0xa6, + 0x52, 0x3b, 0x93, 0x53, 0x6c, 0x00, 0xf7, 0x4e, 0xae, 0x46, 0xc9, 0x72, 0x2c, 0x78, 0xba, 0x52, + 0xbb, 0xa9, 0x38, 0xf3, 0x2a, 0x9b, 0x7d, 0x07, 0x8b, 0x1b, 0xb1, 0x74, 0x69, 0x6a, 0x91, 0x62, + 0x85, 0xcb, 0x0e, 0xa1, 0x77, 0x32, 0xbd, 0x14, 0xe3, 0xb1, 0x18, 0x0f, 0x23, 0x19, 0xf9, 0xed, + 0xba, 0x01, 0xa2, 0xa4, 0xc2, 0xf6, 0x61, 0xeb, 0x55, 0x26, 0x5e, 0x67, 0xd1, 0x6c, 0x91, 0x44, + 0x52, 0x8c, 0xfd, 0x0e, 0x59, 0x2e, 0x33, 0x83, 0x2f, 0x1d, 0xd8, 0xca, 0x31, 0x5a, 0xcc, 0xd3, + 0xd9, 0x42, 0x60, 0x22, 0x9c, 0x64, 0x99, 0x4e, 0x84, 0x93, 0x2c, 0x63, 0x1f, 0x42, 0x8b, 0x8b, + 0xc5, 0x32, 0x91, 0x3a, 0x97, 0x1e, 0x98, 0x73, 0xf5, 0xde, 0x65, 0x22, 0xb9, 0xd6, 0x62, 0x3f, + 0x85, 0xed, 0x52, 0xb6, 0xea, 0xe6, 0xf1, 0x0d, 0xb3, 0xaf, 0x24, 0xe7, 0x15, 0xf5, 0xe0, 0x7f, + 0x4d, 0xe8, 0x5a, 0x96, 0x8b, 0x54, 0x44, 0x14, 0xb7, 0xf2, 0x54, 0x7c, 0x8f, 0x26, 0xbf, 0x1b, + 0x66, 0x23, 0xac, 0x5c, 0x3d, 0x70, 0xce, 0xf2, 0xe4, 0x75, 0xce, 0x4c, 0xb9, 0x74, 0x6f, 0x2b, + 0x97, 0x38, 0x47, 0x7e, 0x1e, 0xcd, 0x26, 0x62, 0x4c, 0xc9, 0xdb, 0xe6, 0x9a, 0x64, 0x07, 0xa6, + 0x76, 0x50, 0xb4, 0x4b, 0x15, 0x49, 0x4b, 0xb8, 0xa9, 0x2f, 0xaa, 0x16, 0xe2, 0xfc, 0xd0, 0x52, + 0x59, 0xa5, 0x28, 0xf6, 0x23, 0xd8, 0xfe, 0x34, 0x19, 0x9b, 0x3a, 0xb7, 0xc8, 0x63, 0xd9, 0x37, + 0xd6, 0x8c, 0x90, 0x57, 0x74, 0xd9, 0xc7, 0xd5, 0x71, 0x8e, 0xa2, 0xda, 0x7d, 0xe2, 0x97, 0xfc, + 0xb7, 0xe4, 0xbc, 0x3a, 0xfe, 0x1d, 0x5a, 0xf3, 0xa5, 0x0f, 0xb4, 0xf9, 0xbe, 0xd9, 0x5c, 0x88, + 0xb8, 0x35, 0x85, 0x7e, 0x64, 0xf7, 0x1d, 0xbf, 0x4b, 0x7b, 0xfa, 0x65, 0xfc, 0x94, 0x8c, 0xdb, + 0xfd, 0xe9, 0xd0, 0x6a, 0x7a, 0x7e, 0xaf, 0x7a, 0x50, 0x21, 0xe2, 0x56, 0x6b, 0x0c, 0x6b, 0x66, + 0x41, 0x7f, 0x8b, 0xb6, 0xd6, 0x0f, 0x7a, 0x4a, 0x85, 0xd7, 0x4c, 0x90, 0x1f, 0x57, 0xa7, 0x06, + 0x7f, 0xbb, 0x0a, 0x54, 0x59, 0xce, 0xab, 0x53, 0xc6, 0xa1, 0x35, 0xb8, 0xfb, 0xf7, 0xaa, 0xf7, + 0x2f, 0x44, 0xdc, 0x1a, 0xef, 0x7f, 0x08, 0x5d, 0x3b, 0xb0, 0x3b, 0xb4, 0xe9, 0x41, 0x5d, 0x60, + 0x17, 0xdc, 0xd6, 0x0c, 0xfe, 0xde, 0x80, 0xad, 0x70, 0x3a, 0x4f, 0x33, 0x69, 0x95, 0x2a, 0xf5, + 0x89, 0xe1, 0xd4, 0x7e, 0x62, 0x34, 0x2a, 0xcd, 0x98, 0x4a, 0x16, 0x95, 0x28, 0x8f, 0x2b, 0xc2, + 0x4a, 0x40, 0xaf, 0x94, 0x80, 0x8f, 0xa0, 0xa3, 0x5e, 0x1b, 0x8a, 0x9a, 0x24, 0x32, 0x0c, 0xf5, + 0xd1, 0xb3, 0xa2, 0x81, 0xb6, 0x45, 0x23, 0xb2, 0x26, 0xb1, 0x3c, 0x2b, 0x35, 0x12, 0xb6, 0x49, + 0x68, 0x71, 0x50, 0xfe, 0x3a, 0x9e, 0x8a, 0x85, 0x8c, 0xa6, 0x73, 0xac, 0x77, 0xee, 0xc0, 0xe5, + 0x16, 0x07, 0x4b, 0x1d, 0x39, 0xf1, 0x3c, 0x13, 0x58, 0x79, 0x8e, 0x24, 0xa5, 0xae, 0xcb, 0x2b, + 0x5c, 0xd4, 0x23, 0xb7, 0x8c, 0x1e, 0x28, 0xbd, 0x32, 0x97, 0xda, 0x75, 0x22, 0xa2, 0x8c, 0x12, + 0xb2, 0xcd, 0x15, 0x11, 0xfc, 0xab, 0x01, 0x4c, 0x21, 0xa9, 0x06, 0xd2, 0xaf, 0x0d, 0xce, 0xdb, + 0x61, 0x2b, 0x83, 0xd3, 0x5a, 0x03, 0xe7, 0xdd, 0x62, 0x8c, 0x56, 0xc0, 0xe4, 0x14, 0xf6, 0x18, + 0xd3, 0xe1, 0x14, 0xaa, 0x0e, 0xb7, 0x59, 0x2c, 0x80, 0x9e, 0xd5, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, + 0x12, 0xaf, 0x06, 0x5a, 0xb8, 0x23, 0xb4, 0xdd, 0xdb, 0xa1, 0xed, 0xd9, 0xd0, 0x7e, 0xe9, 0x40, + 0xef, 0x48, 0xa6, 0xd3, 0x78, 0xc4, 0xc5, 0x28, 0xcd, 0xc6, 0x37, 0x83, 0xaa, 0xe0, 0x6b, 0xd8, + 0xf0, 0x1d, 0x80, 0x1b, 0x7e, 0x91, 0xe5, 0xc5, 0xf7, 0x91, 0x35, 0x00, 0xae, 0xc5, 0x8a, 0xa3, + 0x22, 0x7b, 0x1f, 0x1a, 0x61, 0x46, 0x99, 0x5b, 0x6a, 0x1b, 0xa5, 0x47, 0xc2, 0x1b, 0x61, 0x16, + 0x7c, 0x00, 0x7d, 0x75, 0x29, 0x2d, 0xca, 0xdb, 0x58, 0x1f, 0x9a, 0x27, 0x59, 0x96, 0xea, 0x46, + 0xa6, 0x88, 0xe0, 0x0a, 0xfa, 0x45, 0xf3, 0xc3, 0xc0, 0xbc, 0x4d, 0x7e, 0xd4, 0xfd, 0x51, 0xd8, + 0x83, 0xee, 0x59, 0x2a, 0x3f, 0xcb, 0x62, 0x49, 0xb5, 0x46, 0xf5, 0x0e, 0x9b, 0x15, 0x7c, 0x17, + 0x1e, 0x54, 0x4e, 0x36, 0xfd, 0x16, 0x53, 0xca, 0x35, 0x5f, 0xd7, 0xe7, 0x70, 0xbf, 0x50, 0x0d, + 0x87, 0x6f, 0x75, 0xc7, 0x75, 0xa3, 0xdf, 0xb3, 0x3c, 0x27, 0xa3, 0xf9, 0xf1, 0x35, 0xde, 0x04, + 0xc7, 0xe0, 0xe7, 0x68, 0xaa, 0x1f, 0x1e, 0xf9, 0x0d, 0x2e, 0x62, 0xb1, 0xba, 0xe9, 0xbb, 0x8d, + 0xa6, 0x92, 0x06, 0xfd, 0x26, 0xa1, 0x75, 0xf0, 0xc7, 0x06, 0xf4, 0xeb, 0x8c, 0x98, 0xe4, 0x72, + 0xac, 0xe4, 0x62, 0xcf, 0xa0, 0xf9, 0x45, 0x2c, 0x56, 0x7a, 0xc2, 0x08, 0xd6, 0x42, 0xbe, 0x76, + 0x13, 0xae, 0x36, 0xe0, 0xd3, 0x3a, 0x1a, 0xc9, 0x38, 0x9d, 0xe9, 0x8f, 0x0e, 0x45, 0xe1, 0x39, + 0xc7, 0x49, 0x3a, 0xfa, 0xad, 0xfa, 0x9c, 0xe6, 0x8a, 0xa8, 0x79, 0x2a, 0xcd, 0x3b, 0x3e, 0x95, + 0xcd, 0xda, 0xa7, 0x32, 0x80, 0x7b, 0xbf, 0x9c, 0x8f, 0x23, 0x29, 0x4e, 0xae, 0xe2, 0x85, 0x14, + 0xb3, 0x91, 0xc8, 0x27, 0xb8, 0x2a, 0x3b, 0xf8, 0x9b, 0xa3, 0x51, 0xb5, 0x46, 0xc8, 0xaf, 0x8c, + 0xad, 0x79, 0x4a, 0xae, 0x7e, 0x4a, 0xbe, 0x9a, 0x83, 0xcd, 0xb8, 0xaf, 0x49, 0x9c, 0xbd, 0x71, + 0x49, 0x7f, 0x5d, 0x3c, 0x8a, 0x67, 0x41, 0x7f, 0x45, 0xfd, 0x5a, 0x87, 0x65, 0xb3, 0x0e, 0x96, + 0xe0, 0x57, 0xa5, 0x0e, 0x87, 0x46, 0x8f, 0x26, 0x93, 0x4c, 0x4c, 0x22, 0xa9, 0x33, 0xc2, 0x30, + 0xd8, 0x07, 0xb0, 0x49, 0xca, 0x3a, 0xa8, 0xf5, 0x23, 0x4e, 0xae, 0x73, 0xbc, 0xf3, 0x8f, 0x37, + 0xbb, 0xce, 0x3f, 0xdf, 0xec, 0x3a, 0xff, 0x79, 0xb3, 0xeb, 0xfc, 0xe9, 0xbf, 0xbb, 0x1b, 0x97, + 0x9b, 0xf4, 0xb7, 0xef, 0xfb, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x8c, 0xa8, 0x3e, 0x99, 0xfd, + 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -5152,6 +5162,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.UpdateExistence { + i-- + if m.UpdateExistence { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x38 + } if m.FieldCreatedAt != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) i-- @@ -6400,6 +6420,9 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.FieldCreatedAt != 0 { n += 1 + sovPublic(uint64(m.FieldCreatedAt)) } + if m.UpdateExistence { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -12715,6 +12738,26 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdateExistence", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.UpdateExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 51de362bd..077ed87f9 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -251,6 +251,7 @@ message ImportRoaringRequest { uint64 Block = 4; int64 IndexCreatedAt = 5; int64 FieldCreatedAt = 6; + bool UpdateExistence = 7; } message ImportColumnAttrsRequest { diff --git a/lattice b/lattice index fa773628a..a7c05f2f6 160000 --- a/lattice +++ b/lattice @@ -1 +1 @@ -Subproject commit fa773628a276e2590785a87fbc236c7e88ea6284 +Subproject commit a7c05f2f6aa59d9723403f49de72f5ef0682018a diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 4abe89072..b984d5680 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -15,7 +15,11 @@ package pilosa import ( + "bytes" + "reflect" "testing" + + "github.com/pilosa/pilosa/v2/roaring" ) func TestValidateName(t *testing.T) { @@ -47,10 +51,12 @@ type memAttrStore struct { store map[uint64]map[string]interface{} } -func (s *memAttrStore) Path() string { return "" } -func (s *memAttrStore) Open() error { return nil } -func (s *memAttrStore) Close() error { return nil } -func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } +func (s *memAttrStore) Path() string { return "" } +func (s *memAttrStore) Open() error { return nil } +func (s *memAttrStore) Close() error { return nil } +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return s.store[id], nil +} func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { s.store[id] = m return nil @@ -61,5 +67,28 @@ func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } return nil } -func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } -func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} + +func TestAPI_CombineForExistence(t *testing.T) { + bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 3), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 5), pos(2, 65537), pos(2, 65538)) + buf := new(bytes.Buffer) + _, err := bm.WriteTo(buf) + panicOn(err) + raw := buf.Bytes() + results, err := combineForExistence(raw) + if err != nil { + t.Fatalf("failure to combine: %v", err) + } + bm2 := roaring.NewBitmap() + _, _, err = bm2.ImportRoaringBits(results, false, false, 1< { p.addField(text) } reserved <- '_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field' -posfield <- { p.addPosStr("_field", text) } +posfield <- 'field='? { p.addPosStr("_field", text) } col <- < digits > {p.addPosNum("_col", text)} / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 3956f42dd..c922da38e 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -3466,17 +3466,48 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <( Action53)> */ + /* 18 posfield <- <(('f' 'i' 'e' 'l' 'd' '=')? Action53)> */ func() bool { position388, tokenIndex388 := position, tokenIndex { position389 := position { - position390 := position + position390, tokenIndex390 := position, tokenIndex + if buffer[position] != rune('f') { + goto l390 + } + position++ + if buffer[position] != rune('i') { + goto l390 + } + position++ + if buffer[position] != rune('e') { + goto l390 + } + position++ + if buffer[position] != rune('l') { + goto l390 + } + position++ + if buffer[position] != rune('d') { + goto l390 + } + position++ + if buffer[position] != rune('=') { + goto l390 + } + position++ + goto l391 + l390: + position, tokenIndex = position390, tokenIndex390 + } + l391: + { + position392 := position if !_rules[rulefieldExpr]() { goto l388 } - add(rulePegText, position390) + add(rulePegText, position392) } { add(ruleAction53, position) @@ -3490,175 +3521,153 @@ func (p *PQL) Init(options ...func(*PQL) error) error { }, /* 19 col <- <(( Action54) / (<('\'' singlequotedstring '\'')> Action55) / (<('"' doublequotedstring '"')> Action56))> */ func() bool { - position392, tokenIndex392 := position, tokenIndex + position394, tokenIndex394 := position, tokenIndex { - position393 := position + position395 := position { - position394, tokenIndex394 := position, tokenIndex + position396, tokenIndex396 := position, tokenIndex { - position396 := position + position398 := position if !_rules[ruledigits]() { - goto l395 + goto l397 } - add(rulePegText, position396) + add(rulePegText, position398) } { add(ruleAction54, position) } - goto l394 - l395: - position, tokenIndex = position394, tokenIndex394 + goto l396 + l397: + position, tokenIndex = position396, tokenIndex396 { - position399 := position + position401 := position if buffer[position] != rune('\'') { - goto l398 + goto l400 } position++ if !_rules[rulesinglequotedstring]() { - goto l398 + goto l400 } if buffer[position] != rune('\'') { - goto l398 - } - position++ - add(rulePegText, position399) - } - { - add(ruleAction55, position) - } - goto l394 - l398: - position, tokenIndex = position394, tokenIndex394 - { - position401 := position - if buffer[position] != rune('"') { - goto l392 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l392 - } - if buffer[position] != rune('"') { - goto l392 + goto l400 } position++ add(rulePegText, position401) } + { + add(ruleAction55, position) + } + goto l396 + l400: + position, tokenIndex = position396, tokenIndex396 + { + position403 := position + if buffer[position] != rune('"') { + goto l394 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l394 + } + if buffer[position] != rune('"') { + goto l394 + } + position++ + add(rulePegText, position403) + } { add(ruleAction56, position) } } - l394: - add(rulecol, position393) + l396: + add(rulecol, position395) } return true - l392: - position, tokenIndex = position392, tokenIndex392 + l394: + position, tokenIndex = position394, tokenIndex394 return false }, /* 20 row <- <(( Action57) / (<('\'' singlequotedstring '\'')> Action58) / (<('"' doublequotedstring '"')> Action59))> */ nil, /* 21 open <- <('(' sp)> */ - func() bool { - position404, tokenIndex404 := position, tokenIndex - { - position405 := position - if buffer[position] != rune('(') { - goto l404 - } - position++ - if !_rules[rulesp]() { - goto l404 - } - add(ruleopen, position405) - } - return true - l404: - position, tokenIndex = position404, tokenIndex404 - return false - }, - /* 22 close <- <(sp ')' sp)> */ func() bool { position406, tokenIndex406 := position, tokenIndex { position407 := position - if !_rules[rulesp]() { - goto l406 - } - if buffer[position] != rune(')') { + if buffer[position] != rune('(') { goto l406 } position++ if !_rules[rulesp]() { goto l406 } - add(ruleclose, position407) + add(ruleopen, position407) } return true l406: position, tokenIndex = position406, tokenIndex406 return false }, + /* 22 close <- <(sp ')' sp)> */ + func() bool { + position408, tokenIndex408 := position, tokenIndex + { + position409 := position + if !_rules[rulesp]() { + goto l408 + } + if buffer[position] != rune(')') { + goto l408 + } + position++ + if !_rules[rulesp]() { + goto l408 + } + add(ruleclose, position409) + } + return true + l408: + position, tokenIndex = position408, tokenIndex408 + return false + }, /* 23 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position409 := position - l410: + position411 := position + l412: { - position411, tokenIndex411 := position, tokenIndex + position413, tokenIndex413 := position, tokenIndex { - position412, tokenIndex412 := position, tokenIndex + position414, tokenIndex414 := position, tokenIndex if buffer[position] != rune(' ') { + goto l415 + } + position++ + goto l414 + l415: + position, tokenIndex = position414, tokenIndex414 + if buffer[position] != rune('\t') { + goto l416 + } + position++ + goto l414 + l416: + position, tokenIndex = position414, tokenIndex414 + if buffer[position] != rune('\n') { goto l413 } position++ - goto l412 - l413: - position, tokenIndex = position412, tokenIndex412 - if buffer[position] != rune('\t') { - goto l414 - } - position++ - goto l412 - l414: - position, tokenIndex = position412, tokenIndex412 - if buffer[position] != rune('\n') { - goto l411 - } - position++ } - l412: - goto l410 - l411: - position, tokenIndex = position411, tokenIndex411 + l414: + goto l412 + l413: + position, tokenIndex = position413, tokenIndex413 } - add(rulesp, position409) + add(rulesp, position411) } return true }, /* 24 eq <- <(sp '=' sp)> */ - func() bool { - position415, tokenIndex415 := position, tokenIndex - { - position416 := position - if !_rules[rulesp]() { - goto l415 - } - if buffer[position] != rune('=') { - goto l415 - } - position++ - if !_rules[rulesp]() { - goto l415 - } - add(ruleeq, position416) - } - return true - l415: - position, tokenIndex = position415, tokenIndex415 - return false - }, - /* 25 comma <- <(sp ',' sp)> */ func() bool { position417, tokenIndex417 := position, tokenIndex { @@ -3666,298 +3675,302 @@ func (p *PQL) Init(options ...func(*PQL) error) error { if !_rules[rulesp]() { goto l417 } - if buffer[position] != rune(',') { + if buffer[position] != rune('=') { goto l417 } position++ if !_rules[rulesp]() { goto l417 } - add(rulecomma, position418) + add(ruleeq, position418) } return true l417: position, tokenIndex = position417, tokenIndex417 return false }, + /* 25 comma <- <(sp ',' sp)> */ + func() bool { + position419, tokenIndex419 := position, tokenIndex + { + position420 := position + if !_rules[rulesp]() { + goto l419 + } + if buffer[position] != rune(',') { + goto l419 + } + position++ + if !_rules[rulesp]() { + goto l419 + } + add(rulecomma, position420) + } + return true + l419: + position, tokenIndex = position419, tokenIndex419 + return false + }, /* 26 lbrack <- <('[' sp)> */ nil, /* 27 rbrack <- <(sp ']' sp)> */ nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position421, tokenIndex421 := position, tokenIndex + position423, tokenIndex423 := position, tokenIndex { - position422 := position + position424 := position { - position423, tokenIndex423 := position, tokenIndex + position425, tokenIndex425 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l424 + goto l426 } position++ - goto l423 - l424: - position, tokenIndex = position423, tokenIndex423 + goto l425 + l426: + position, tokenIndex = position425, tokenIndex425 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l421 + goto l423 } position++ } - l423: l425: + l427: { - position426, tokenIndex426 := position, tokenIndex + position428, tokenIndex428 := position, tokenIndex { - position427, tokenIndex427 := position, tokenIndex + position429, tokenIndex429 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l430 + } + position++ + goto l429 + l430: + position, tokenIndex = position429, tokenIndex429 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l431 + } + position++ + goto l429 + l431: + position, tokenIndex = position429, tokenIndex429 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l428 } position++ - goto l427 - l428: - position, tokenIndex = position427, tokenIndex427 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l429 - } - position++ - goto l427 - l429: - position, tokenIndex = position427, tokenIndex427 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l426 - } - position++ } - l427: - goto l425 - l426: - position, tokenIndex = position426, tokenIndex426 + l429: + goto l427 + l428: + position, tokenIndex = position428, tokenIndex428 } - add(ruleIDENT, position422) + add(ruleIDENT, position424) } return true - l421: - position, tokenIndex = position421, tokenIndex421 + l423: + position, tokenIndex = position423, tokenIndex423 return false }, /* 29 digits <- <[0-9]+> */ func() bool { - position430, tokenIndex430 := position, tokenIndex + position432, tokenIndex432 := position, tokenIndex { - position431 := position + position433 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l430 + goto l432 } position++ - l432: + l434: { - position433, tokenIndex433 := position, tokenIndex + position435, tokenIndex435 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l433 + goto l435 } position++ - goto l432 - l433: - position, tokenIndex = position433, tokenIndex433 + goto l434 + l435: + position, tokenIndex = position435, tokenIndex435 } - add(ruledigits, position431) + add(ruledigits, position433) } return true - l430: - position, tokenIndex = position430, tokenIndex430 + l432: + position, tokenIndex = position432, tokenIndex432 return false }, /* 30 signedDigits <- <('-'? digits)> */ nil, /* 31 decimal <- <((signedDigits ('.' digits?)?) / ('-'? '.' digits))> */ func() bool { - position435, tokenIndex435 := position, tokenIndex + position437, tokenIndex437 := position, tokenIndex { - position436 := position + position438 := position { - position437, tokenIndex437 := position, tokenIndex + position439, tokenIndex439 := position, tokenIndex { - position439 := position + position441 := position { - position440, tokenIndex440 := position, tokenIndex + position442, tokenIndex442 := position, tokenIndex if buffer[position] != rune('-') { - goto l440 + goto l442 } position++ - goto l441 - l440: - position, tokenIndex = position440, tokenIndex440 + goto l443 + l442: + position, tokenIndex = position442, tokenIndex442 } - l441: + l443: if !_rules[ruledigits]() { - goto l438 + goto l440 } - add(rulesignedDigits, position439) + add(rulesignedDigits, position441) } { - position442, tokenIndex442 := position, tokenIndex + position444, tokenIndex444 := position, tokenIndex if buffer[position] != rune('.') { - goto l442 + goto l444 } position++ { - position444, tokenIndex444 := position, tokenIndex + position446, tokenIndex446 := position, tokenIndex if !_rules[ruledigits]() { - goto l444 + goto l446 } - goto l445 - l444: - position, tokenIndex = position444, tokenIndex444 + goto l447 + l446: + position, tokenIndex = position446, tokenIndex446 } - l445: - goto l443 - l442: - position, tokenIndex = position442, tokenIndex442 + l447: + goto l445 + l444: + position, tokenIndex = position444, tokenIndex444 } - l443: - goto l437 - l438: - position, tokenIndex = position437, tokenIndex437 + l445: + goto l439 + l440: + position, tokenIndex = position439, tokenIndex439 { - position446, tokenIndex446 := position, tokenIndex + position448, tokenIndex448 := position, tokenIndex if buffer[position] != rune('-') { - goto l446 + goto l448 } position++ - goto l447 - l446: - position, tokenIndex = position446, tokenIndex446 + goto l449 + l448: + position, tokenIndex = position448, tokenIndex448 } - l447: + l449: if buffer[position] != rune('.') { - goto l435 + goto l437 } position++ if !_rules[ruledigits]() { - goto l435 + goto l437 } } - l437: - add(ruledecimal, position436) + l439: + add(ruledecimal, position438) } return true - l435: - position, tokenIndex = position435, tokenIndex435 + l437: + position, tokenIndex = position437, tokenIndex437 return false }, /* 32 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position448, tokenIndex448 := position, tokenIndex + position450, tokenIndex450 := position, tokenIndex { - position449 := position + position451 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if buffer[position] != rune('-') { - goto l448 + goto l450 } position++ { - position450, tokenIndex450 := position, tokenIndex + position452, tokenIndex452 := position, tokenIndex if buffer[position] != rune('0') { - goto l451 + goto l453 } position++ - goto l450 - l451: - position, tokenIndex = position450, tokenIndex450 + goto l452 + l453: + position, tokenIndex = position452, tokenIndex452 if buffer[position] != rune('1') { - goto l448 + goto l450 } position++ } - l450: + l452: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if buffer[position] != rune('-') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if buffer[position] != rune('T') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if buffer[position] != rune(':') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l448 + goto l450 } position++ - add(ruletimestampbasicfmt, position449) + add(ruletimestampbasicfmt, position451) } return true - l448: - position, tokenIndex = position448, tokenIndex448 + l450: + position, tokenIndex = position450, tokenIndex450 return false }, /* 33 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position452, tokenIndex452 := position, tokenIndex + position454, tokenIndex454 := position, tokenIndex { - position453 := position + position455 := position { - position454, tokenIndex454 := position, tokenIndex + position456, tokenIndex456 := position, tokenIndex if buffer[position] != rune('"') { - goto l455 - } - position++ - { - position456 := position - if !_rules[ruletimestampbasicfmt]() { - goto l455 - } - add(rulePegText, position456) - } - if buffer[position] != rune('"') { - goto l455 - } - position++ - goto l454 - l455: - position, tokenIndex = position454, tokenIndex454 - if buffer[position] != rune('\'') { goto l457 } position++ @@ -3968,27 +3981,45 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } add(rulePegText, position458) } - if buffer[position] != rune('\'') { + if buffer[position] != rune('"') { goto l457 } position++ - goto l454 + goto l456 l457: - position, tokenIndex = position454, tokenIndex454 + position, tokenIndex = position456, tokenIndex456 + if buffer[position] != rune('\'') { + goto l459 + } + position++ { - position459 := position + position460 := position if !_rules[ruletimestampbasicfmt]() { - goto l452 + goto l459 } - add(rulePegText, position459) + add(rulePegText, position460) + } + if buffer[position] != rune('\'') { + goto l459 + } + position++ + goto l456 + l459: + position, tokenIndex = position456, tokenIndex456 + { + position461 := position + if !_rules[ruletimestampbasicfmt]() { + goto l454 + } + add(rulePegText, position461) } } - l454: - add(ruletimestampfmt, position453) + l456: + add(ruletimestampfmt, position455) } return true - l452: - position, tokenIndex = position452, tokenIndex452 + l454: + position, tokenIndex = position454, tokenIndex454 return false }, /* 34 timestamp <- <( Action60)> */ diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 12b9e4b15..9d3d1192e 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -425,6 +425,50 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row", Args: map[string]interface{}{"unicode": "Æ�漢д ☮♬ ♞🜻💣"}}, }, }}, + { + name: "TopK", + call: "TopK(myfield, Row(), k=7)", + exp: &Call{ + Name: "TopK", + Args: map[string]interface{}{ + "_field": "myfield", + "k": int64(7), + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "TopKWithField=", + call: "TopK(field=myfield, Row(), k=7)", + exp: &Call{ + Name: "TopK", + Args: map[string]interface{}{ + "_field": "myfield", + "k": int64(7), + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "Rows", + call: "Rows(myfield)", + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "_field": "myfield", + }, + }}, + { + name: "RowsWithField=", + call: "Rows(field=myfield)", + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "_field": "myfield", + }, + }}, { name: "SetRowAttrs", call: "SetRowAttrs(myfield, 9, z=4)", @@ -436,6 +480,17 @@ func TestPQLDeepEquality(t *testing.T) { "_row": int64(9), }, }}, + { + name: "SetRowAttrsWithField=", + call: "SetRowAttrs(field=myfield, 9, z=4)", + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, { name: "SetRowAttrsWithRowKeySingleQuote", call: "SetRowAttrs(myfield, 'rowKey', z=4)", @@ -522,6 +577,19 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row"}, }, }}, + { + name: "TopNwithField=", + call: "TopN(field=myfield, Row(), a=7)", + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, { name: "RangeEQ", call: "Row(a==7)", diff --git a/rbf.go b/rbf.go index f309f3f5a..874d996f4 100644 --- a/rbf.go +++ b/rbf.go @@ -195,7 +195,6 @@ type RBFTx struct { // initialIndex is only a debugging aid. Transactions // can cross indexes. It can be left empty without consequence. initialIndex string - frag *fragment tx *rbf.Tx o Txo sn int64 // serial number @@ -531,7 +530,6 @@ func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err rtx := &RBFTx{ tx: tx, initialIndex: initialIndex, - frag: o.Fragment, o: o, sn: sn, Db: w, diff --git a/roaring/roaring.go b/roaring/roaring.go index 9c3a97aa9..1ee979bde 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2280,6 +2280,51 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui return b.ImportRoaringRawIterator(itr, clear, log, rowSize) } +// MergeRoaringRawIteratorIntoExists is a special merge iterator that flattens the results onto a +// single row which is used for determining existence. All row references are removed and only the column +// is considered. +func (b *Bitmap) MergeRoaringRawIteratorIntoExists(itr RoaringIterator, rowSize uint64) error { + + if itr == nil { + return errors.New("nil RoaringIterator passed to MergeRoaringRawIteratorIntoExists") + } + var synthC Container + importUpdater := func(oldC *Container, existed bool) (newC *Container, write bool) { + existN := oldC.N() + if existN == MaxContainerVal+1 { + return oldC, false + } + if existN == 0 { + newerC := synthC.Clone() + return newerC, true + } + newC = oldC.unionInPlace(&synthC) + if newC.typeID == ContainerBitmap { + newC.Repair() + } + if newC.N() != existN { + return newC, true + } + return oldC, false + } + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := itr.Next() + for itrErr == nil { + synthC.typeID = itrCType + synthC.n = int32(itrN) + synthC.len = int32(itrLen) + synthC.cap = int32(itrLen) + synthC.pointer = itrPointer + b.Containers.Update(itrKey%rowSize, importUpdater) + itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() + } + // note: if we get a non-EOF err, it's possible that we made SOME + // changes but didn't log them. I don't have a good solution to this. + if itrErr != io.EOF { + return itrErr + } + return nil +} + func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { var itrKey uint64 var itrCType byte diff --git a/scripts/bench_write.sh b/scripts/bench_write.sh index 49ec38101..f9b18ed8e 100755 --- a/scripts/bench_write.sh +++ b/scripts/bench_write.sh @@ -21,7 +21,7 @@ SHA=$(git -C $PILOSA_SRC rev-parse HEAD) # Format current date. DATE=$(date '+%Y%m%d') -for FILENAME in gh.1m.yml gh.issues.keyed.yml gh.issues.unkeyed.yml +for FILENAME in gh.1m.yml gh.issues.keyed.yml gh.issues.unkeyed.yml gh.issues.autogenerate.yml do WORKFLOW_PATH="${BASH_SOURCE%/*}/etc/gloat/${FILENAME}" WORKFLOW_NAME="$(gloat workflow name $WORKFLOW_PATH)" diff --git a/scripts/etc/gloat/gh.issues.autogenerate.yml b/scripts/etc/gloat/gh.issues.autogenerate.yml new file mode 100644 index 000000000..d1ceeace5 --- /dev/null +++ b/scripts/etc/gloat/gh.issues.autogenerate.yml @@ -0,0 +1,9 @@ +name: "GitHub Issues Import Load Testing (two weeks, autogenerated ID)" + +main: "pilosa server --data-dir ${TMPDIR} --txsrc ${STORAGE_BACKEND}" +load: "molecula-consumer-github -i issues -a --external-generate --record-type issue --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-13T23:00:00Z --cache-dir ~/.githubarchive" + +health_url: "http://localhost:10101/status" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/server/config.go b/server/config.go index 29c3434a6..6ae013bab 100644 --- a/server/config.go +++ b/server/config.go @@ -33,8 +33,9 @@ import ( ) const ( - defaultBindPort = "10101" - defaultBindGRPCPort = "20101" + defaultBindPort = "10101" + defaultBindGRPCPort = "20101" + defaultDiagnosticsInterval = 1 * time.Hour ) // TLSConfig contains TLS configuration diff --git a/server/default.go b/server/default.go deleted file mode 100644 index ce2fe8aaa..000000000 --- a/server/default.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build !release -// -// This file sets defaults to be overridden by release.go - -package server - -import "time" - -// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. -const defaultDiagnosticsInterval = time.Duration(0) diff --git a/server/release.go b/server/release.go deleted file mode 100644 index d988f4f81..000000000 --- a/server/release.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build release -// -// This file sets release-specific variables. - -package server - -import "time" - -// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. -const defaultDiagnosticsInterval = 1 * time.Hour