diff --git a/.circleci/config.yml b/.circleci/config.yml index c696d5691..c7fe6e11b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,12 +24,18 @@ jobs: - persist_to_workspace: root: . paths: "*" + check-license-headers: + <<: *defaults + steps: + - *fast-checkout + - run: make check-license-headers linter: <<: *defaults steps: - *fast-checkout - - run: make install-gometalinter - - run: make gometalinter + - run: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.13.2 + - run: sudo cp bin/golangci-lint /usr/local/bin/ + - run: make golangci-lint test-build-arm: <<: *defaults steps: @@ -44,6 +50,12 @@ jobs: - *fast-checkout - run: sudo apt-get install lsof - run: make test + test-golang-1.12-shard22: &base-test + <<: *defaults + steps: + - *fast-checkout + - run: sudo apt-get install lsof + - run: make test SHARD_WIDTH=22 test-golang-1.12-race: <<: *defaults steps: @@ -117,6 +129,9 @@ workflows: - linter: requires: - setup + - check-license-headers: + requires: + - setup - test-build-arm: requires: - setup @@ -138,10 +153,12 @@ workflows: - prerelease: requires: - linter + - check-license-headers - test-golang-1.12 - release: requires: - linter + - check-license-headers - test-golang-1.12 filters: tags: @@ -154,4 +171,5 @@ workflows: - dockerhub-upload: requires: - linter + - check-license-headers - test-golang-1.12 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 081a20073..ceb76f05f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,6 +12,8 @@ Fixes # - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. +- [ ] Make sure PR title conforms to convention in CHANGELOG.md. +- [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for. @@ -22,3 +24,5 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. +- [ ] Make sure PR title conforms to convention in CHANGELOG.md. +- [ ] Make sure PR is tagged with appropriate changelog label. diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b558ac7..531116f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,68 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.3.0] - 2019-04-16 + +This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. + +### Added +- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) +- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) +- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) +- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) +- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) +- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) +- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) +- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) +- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) +- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) +- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) +- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) +- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) +- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) +- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) +- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) + +### Fixed +- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) +- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) +- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) +- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) +- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) +- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) +- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) +- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) +- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) +- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) +- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) +- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) +- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) +- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) +- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) +- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) +- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) +- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) + +### Changed +- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) +- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) +- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) +- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) +- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) +- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) +- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) +- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) + +### Performance +- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) +- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) +- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) +- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) +- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) +- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) +- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) +- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) + ## [1.2.0] - 2018-12-20 This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93e4854f5..8e4b45e4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,11 +28,17 @@ If you want to help but you aren't sure where to start, check out our [github la - Fork the [Pilosa repository][2] to your own account. -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: +- It will be easier to follow these instructions if you: + + ```sh + export GH_USERNAME= + ``` + +- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: ```sh mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone git@github.com:${USER}/pilosa.git + git clone https://github.com/pilosa/pilosa.git ``` - `cd` to your pilosa directory: @@ -46,20 +52,14 @@ If you want to help but you aren't sure where to start, check out our [github la ```sh make install ``` - - or - - ``` - go install github.com/pilosa/pilosa/cmd/... - ``` Running `pilosa` should now run a Pilosa instance. -- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: +- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username ```sh cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add upstream git@github.com:pilosa/pilosa.git + git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git ``` ### Makefile @@ -133,9 +133,8 @@ Additional commands are available in the `Makefile`. - Before starting to work on a task, sync your branch with the upstream: ```sh - git fetch upstream git checkout master - git merge upstream/master + git pull ``` - Create a local feature branch: @@ -155,15 +154,16 @@ Additional commands are available in the `Makefile`. - Verify that your pull request is applied to the latest version of code on github: ```sh - git remote add upstream git@github.com:pilosa/pilosa.git - git fetch upstream - git rebase -i upstream/master + git checkout master + git pull + git checkout something-amazing + git rebase master ``` - Push to your fork: ```sh - git push -u something-amazing + git push -u $GH_USERNAME something-amazing:something-amazing ``` - Submit a [pull request][3] diff --git a/Dockerfile b/Dockerfile index 9914401d8..afbda01f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.11.5 as builder +FROM golang:1.12.4 as builder COPY . pilosa diff --git a/Makefile b/Makefile index 45b1df78c..9757af157 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) +SHARD_WIDTH = 20 LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)" GO_VERSION=latest ENTERPRISE ?= 0 @@ -14,6 +15,8 @@ RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) +BUILD_TAGS += shardwidth$(SHARD_WIDTH) +LICENSE_HASH=$(shell head -13 pilosa.go | shasum | cut -f 1 -d " ") export GO111MODULE=on # Run tests and compile Pilosa @@ -29,7 +32,7 @@ vendor: go.mod # Run test suite test: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) + go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) bench: go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) @@ -128,6 +131,10 @@ docker-build: docker-test: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./... +# Run golangci-lint +golangci-lint: require-golangci-lint + golangci-lint run + # Run gometalinter with custom flags gometalinter: require-gometalinter vendor GO111MODULE=off gometalinter --vendor --disable-all \ @@ -151,6 +158,13 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... +# Verify that all Go files have license header +check-license-headers: SHELL:=/bin/bash +check-license-headers: + @! find . -name '*.go' | grep -v '^./vendor' | while read fn;\ + do [[ `head -13 $$fn | shasum | cut -f 1 -d " "` == $(LICENSE_HASH) ]] || echo $$fn; done | \ + grep -v apimethod_string.go | grep -v pb.go | grep -v peg.go | grep -v lru.go | grep -v btree | grep -v enterprise + ###################### # Build dependencies # ###################### @@ -175,6 +189,9 @@ install-protoc: install-peg: GO111MODULE=off go get github.com/pointlander/peg +install-golangci-lint: + GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install diff --git a/api.go b/api.go index 19e41732d..bdaa35b03 100644 --- a/api.go +++ b/api.go @@ -96,7 +96,7 @@ func (api *API) validate(f apiMethod) error { if _, ok := validAPIMethods[state][f]; ok { return nil } - return newApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) + return newAPIMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) } // Query parses a PQL query out of the request and executes it. @@ -213,7 +213,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str for _, opt := range opts { err := opt(&fo) if err != nil { - return nil, errors.Wrap(err, "applying option") + return nil, NewBadRequestError(errors.Wrap(err, "applying option")) } } @@ -290,6 +290,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { // bitmap. func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") + span.LogKV("index", indexName, "field", fieldName) defer span.Finish() if err = api.validate(apiField); err != nil { @@ -325,7 +326,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err = field.importRoaring(viewData, shard, viewName, req.Clear) + err = field.importRoaring(ctx, viewData, shard, viewName, req.Clear) if err != nil { return errors.Wrap(err, "importing pilosa roaring") } @@ -335,7 +336,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, // field.importRoaring changes the standard roaring run format to pilosa roaring data := make([]byte, len(viewData)) copy(data, viewData) - err = field.importRoaring(data, shard, viewName, req.Clear) + err = field.importRoaring(ctx, data, shard, viewName, req.Clear) if err != nil { return errors.Wrap(err, "importing standard roaring") } @@ -617,7 +618,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return nil } -// PostClusterMessage is for internal use. It decodes a protobuf message out of +// ClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { span, _ := tracing.StartSpanFromContext(ctx, "API.ClusterMessage") @@ -655,6 +656,33 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.holder.limitedSchema() } +// ApplySchema takes the given schema and applies it across the +// cluster (if remote is false), or just to this node (if remote is +// true). This is designed for the use case of replicating a schema +// from one Pilosa cluster to another which is initially empty. It is +// not officially supported in other scenarios and may produce +// surprising results. +func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.ApplySchema") + defer span.Finish() + + if err := api.validate(apiApplySchema); err != nil { + return errors.Wrap(err, "validating api method") + } + + if !remote { + nodes := api.cluster.Nodes() + for i, node := range nodes { + err := api.server.defaultClient.PostSchema(ctx, &node.URI, s, true) + if err != nil { + return errors.Wrapf(err, "forwarding post schema to node %d of %d", i+1, len(nodes)) + } + } + } + + return api.holder.applySchema(s) +} + // Views returns the views in the given field. func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Views") @@ -712,7 +740,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri return errors.Wrap(err, "sending DeleteView message") } -// IndexAttrDiff +// IndexAttrDiff determines the local column attribute data blocks which differ from those provided. func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff") defer span.Finish() @@ -750,6 +778,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At return attrs, nil } +// FieldAttrDiff determines the local row attribute data blocks which differ from those provided. func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff") defer span.Finish() @@ -796,6 +825,8 @@ type ImportOptions struct { // ImportOption is a functional option type for API.Import. type ImportOption func(*ImportOptions) error +// OptImportOptionsClear is a functional option on ImportOption +// used to specify whether the import is a set or clear operation. func OptImportOptionsClear(c bool) ImportOption { return func(o *ImportOptions) error { o.Clear = c @@ -803,6 +834,8 @@ func OptImportOptionsClear(c bool) ImportOption { } } +// OptImportOptionsIgnoreKeyCheck is a functional option on ImportOption +// used to specify whether key check should be ignored. func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption { return func(o *ImportOptions) error { o.IgnoreKeyCheck = b @@ -864,11 +897,14 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if _, ok := m[shard]; !ok { m[shard] = make([]Bit, 0) } - m[shard] = append(m[shard], Bit{ - RowID: req.RowIDs[i], - ColumnID: colID, - Timestamp: req.Timestamps[i], - }) + bit := Bit{ + RowID: req.RowIDs[i], + ColumnID: colID, + } + if len(req.Timestamps) > 0 { + bit.Timestamp = req.Timestamps[i] + } + m[shard] = append(m[shard], bit) } // Signal to the receiving nodes to ignore checking for key translation. @@ -1175,7 +1211,7 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } -// Info returns information about this server instance +// Info returns information about this server instance. func (api *API) Info() serverInfo { si := api.server.systemInfo // we don't report errors on failures to get this information @@ -1192,6 +1228,7 @@ func (api *API) Info() serverInfo { } } +// TranslateKeys handles a TranslateKeyRequest. func (api *API) TranslateKeys(body io.Reader) ([]byte, error) { reqBytes, err := ioutil.ReadAll(body) if err != nil { @@ -1267,6 +1304,7 @@ const ( //apiStatsWithTags // not implemented //apiVersion // not implemented apiViews + apiApplySchema ) var methodsCommon = map[apiMethod]struct{}{ @@ -1300,4 +1338,5 @@ var methodsNormal = map[apiMethod]struct{}{ apiRemoveNode: {}, apiShardNodes: {}, apiViews: {}, + apiApplySchema: {}, } diff --git a/api_test.go b/api_test.go index a33b064c1..0cde55d59 100644 --- a/api_test.go +++ b/api_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "context" "fmt" + "math" "reflect" "strings" "testing" @@ -198,7 +199,7 @@ func TestAPI_ImportValue(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(0, 100)) + _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatalf("creating field: %v", err) } diff --git a/apimethod_string.go b/apimethod_string.go index cafeab5af..309648a62 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,40 @@ package pilosa import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[apiClusterMessage-0] + _ = x[apiCreateField-1] + _ = x[apiCreateIndex-2] + _ = x[apiDeleteField-3] + _ = x[apiDeleteAvailableShard-4] + _ = x[apiDeleteIndex-5] + _ = x[apiDeleteView-6] + _ = x[apiExportCSV-7] + _ = x[apiFragmentBlockData-8] + _ = x[apiFragmentBlocks-9] + _ = x[apiFragmentData-10] + _ = x[apiField-11] + _ = x[apiFieldAttrDiff-12] + _ = x[apiImport-13] + _ = x[apiImportValue-14] + _ = x[apiIndex-15] + _ = x[apiIndexAttrDiff-16] + _ = x[apiQuery-17] + _ = x[apiRecalculateCaches-18] + _ = x[apiRemoveNode-19] + _ = x[apiResizeAbort-20] + _ = x[apiSetCoordinator-21] + _ = x[apiShardNodes-22] + _ = x[apiViews-23] + _ = x[apiApplySchema-24] +} -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 166, 182, 191, 205, 213, 229, 237, 257, 270, 284, 301, 314, 322} +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchema" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 181, 197, 206, 220, 228, 244, 252, 272, 285, 299, 316, 329, 337, 351} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 260892646..e8c770fde 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -215,66 +215,64 @@ func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } // Blocks returns a list of all blocks in the store. -func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) { - tx, err := s.db.Begin(false) - if err != nil { - return nil, errors.Wrap(err, "starting transaction") - } - defer tx.Rollback() +func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) { + err = s.db.View(func(tx *bolt.Tx) error { + // Wrap cursor to segment by block. + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) - // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) + // Iterate over each block. + for cur.nextBlock() { + block := pilosa.AttrBlock{ID: cur.blockID()} - // Iterate over each block. - var blocks []pilosa.AttrBlock - for cur.nextBlock() { - block := pilosa.AttrBlock{ID: cur.blockID()} + // Compute checksum of every key/value in block. + h := xxhash.New() + for k, v := cur.next(); k != nil; k, v = cur.next() { + // hash function writes don't usually need to be checked + _, _ = h.Write(k) + _, _ = h.Write(v) + } + block.Checksum = h.Sum(nil) - // Compute checksum of every key/value in block. - h := xxhash.New() - for k, v := cur.next(); k != nil; k, v = cur.next() { - h.Write(k) - h.Write(v) + // Append block. + blocks = append(blocks, block) } - block.Checksum = h.Sum(nil) - - // Append block. - blocks = append(blocks, block) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "getting blocks") } - return blocks, nil } // BlockData returns all data for a single block. -func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { - m := make(map[uint64]map[string]interface{}) +func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) { + m = make(map[uint64]map[string]interface{}) // Start read-only transaction. - tx, err := s.db.Begin(false) + err = s.db.View(func(tx *bolt.Tx) error { + // Move to the start of the block. + min := u64tob(i * attrBlockSize) + max := u64tob((i + 1) * attrBlockSize) + cur := tx.Bucket([]byte("attrs")).Cursor() + for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { + // Exit if we're past the end of the block. + if bytes.Compare(k, max) != -1 { + break + } + + // Decode attribute map and associate with id. + attrs, err := pilosa.DecodeAttrs(v) + if err != nil { + return errors.Wrap(err, "decoding attrs") + } + m[btou64(k)] = attrs + + } + return nil + }) if err != nil { - return nil, errors.Wrap(err, "starting transaction") + return nil, errors.Wrap(err, "getting block data") } - defer tx.Rollback() - - // Move to the start of the block. - min := u64tob(i * attrBlockSize) - max := u64tob((i + 1) * attrBlockSize) - cur := tx.Bucket([]byte("attrs")).Cursor() - for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { - // Exit if we're past the end of the block. - if bytes.Compare(k, max) != -1 { - break - } - - // Decode attribute map and associate with id. - attrs, err := pilosa.DecodeAttrs(v) - if err != nil { - return nil, errors.Wrap(err, "decoding attrs") - } - m[btou64(k)] = attrs - - } - return m, nil } diff --git a/cache.go b/cache.go index 40509ab64..0bf4cd09d 100644 --- a/cache.go +++ b/cache.go @@ -452,9 +452,14 @@ func (s *simpleCache) Fetch(id uint64) (*Row, bool) { return m, ok } -// Add adds the bitmap to the cache, keyed on the id. +// Add adds the bitmap to the cache, keyed on the id. A nil row means +// deleting the row from the cache. func (s *simpleCache) Add(id uint64, b *Row) { - s.cache[id] = b + if b != nil { + s.cache[id] = b + } else { + delete(s.cache, id) + } } // nopCache represents a no-op Cache implementation. diff --git a/client.go b/client.go index 3762a27b9..3b815a6bd 100644 --- a/client.go +++ b/client.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -32,6 +46,7 @@ type FieldValue struct { type InternalClient interface { MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) + PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Nodes(ctx context.Context) ([]*Node, error) @@ -58,6 +73,7 @@ type InternalClient interface { //=============== +// InternalQueryClient is the internal interface for querying a node. type InternalQueryClient interface { QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } @@ -88,6 +104,10 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, return nil, nil } func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } +func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error { + return nil +} + func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } diff --git a/cluster.go b/cluster.go index fc332df09..a48b12dc5 100644 --- a/cluster.go +++ b/cluster.go @@ -848,8 +848,8 @@ func (c *cluster) partition(index string, shard uint64) int { // Hash the bytes and mod by partition count. h := fnv.New64a() - h.Write([]byte(index)) - h.Write(buf[:]) + _, _ = h.Write([]byte(index)) + _, _ = h.Write(buf[:]) return int(h.Sum64() % uint64(c.partitionN)) } @@ -1915,7 +1915,12 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { for _, node := range officialNodes { if node.ID == c.Node.ID && node.State != c.Node.State { c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State) - go c.setNodeState(c.Node.State) + go func(fromState, toState string) { + err := c.setNodeState(toState) + if err != nil { + c.logger.Printf("error setting node state from %v to %v: %v", fromState, toState, err) + } + }(node.State, c.Node.State) } if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") @@ -1987,12 +1992,16 @@ func (c *cluster) setStatic(hosts []string) error { return nil } +// ClusterStatus describes the status of the cluster including its +// state and node topology. type ClusterStatus struct { ClusterID string State string Nodes []*Node } +// ResizeInstruction contains the instruction provided to a node +// during a cluster resize operation. type ResizeInstruction struct { JobID int64 Node *Node @@ -2002,6 +2011,8 @@ type ResizeInstruction struct { ClusterStatus *ClusterStatus } +// ResizeSource is the source of data for a node acting on a +// ResizeInstruction. type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -2041,82 +2052,101 @@ func decodeTopology(topology *internal.Topology) (*Topology, error) { return t, nil } +// CreateShardMessage is an internal message indicating shard creation. type CreateShardMessage struct { Index string Field string Shard uint64 } +// CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { Index string Meta *IndexOptions } +// DeleteIndexMessage is an internal message indicating index deletion. type DeleteIndexMessage struct { Index string } +// CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { Index string Field string Meta *FieldOptions } +// DeleteFieldMessage is an internal message indicating field deletion. type DeleteFieldMessage struct { Index string Field string } +// DeleteAvailableShardMessage is an internal message indicating available shard deletion. type DeleteAvailableShardMessage struct { Index string Field string ShardID uint64 } +// CreateViewMessage is an internal message indicating view creation. type CreateViewMessage struct { Index string Field string View string } + +// DeleteViewMessage is an internal message indicating view deletion. type DeleteViewMessage struct { Index string Field string View string } +// ResizeInstructionComplete is an internal message to the coordinator indicating +// that the resize instructions performed on a single node have completed. type ResizeInstructionComplete struct { JobID int64 Node *Node Error string } +// SetCoordinatorMessage is an internal message instructing nodes to honor a new coordinator. type SetCoordinatorMessage struct { New *Node } +// UpdateCoordinatorMessage is an internal message for reassigning the coordinator. type UpdateCoordinatorMessage struct { New *Node } +// NodeStateMessage is an internal message for broadcasting a node's state. type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } +// NodeStatus is an internal message representing the contents of a node. type NodeStatus struct { Node *Node Indexes []*IndexStatus Schema *Schema } +// IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { Name string Fields []*FieldStatus } +// FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string AvailableShards *roaring.Bitmap } +// RecalculateCaches is an internal message for recalculating all caches +// within a holder. type RecalculateCaches struct{} diff --git a/cluster_internal_test.go b/cluster_internal_test.go index a6cf437b2..2a557694d 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 1300000, nil) + _, err = field.SetBit(1, ShardWidth+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 2600000, nil) + _, err = field.SetBit(1, ShardWidth*2+1, nil) if err != nil { t.Fatal(err) } - _, err = field.SetBit(1, 3900000, nil) + _, err = field.SetBit(1, ShardWidth*3+1, nil) if err != nil { t.Fatal(err) } @@ -607,7 +607,9 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node := tc.Clusters[0] @@ -615,7 +617,9 @@ func TestCluster_ResizeStates(t *testing.T) { top := &Topology{ nodeIDs: []string{node.Node.ID}, } - tc.WriteTopology(node.Path, top) + if err := tc.WriteTopology(node.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { @@ -635,7 +639,9 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, not in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node := tc.Clusters[0] @@ -643,7 +649,9 @@ func TestCluster_ResizeStates(t *testing.T) { top := &Topology{ nodeIDs: []string{"some-other-host"}, } - tc.WriteTopology(node.Path, top) + if err := tc.WriteTopology(node.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. expected := "coordinator node0 is not in topology: [some-other-host]" @@ -660,14 +668,18 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, no data", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { - t.Fatal(err) + t.Fatalf("opening cluster: %v", err) } - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] node1 := tc.Clusters[1] @@ -698,18 +710,22 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] // write topology to data file top := &Topology{ nodeIDs: []string{"node0", "node2"}, } - tc.WriteTopology(node0.Path, top) + if err := tc.WriteTopology(node0.Path, top); err != nil { + t.Fatalf("writing topology: %v", err) + } // Open TestCluster. if err := tc.Open(); err != nil { - t.Fatal(err) + t.Fatalf("opening cluster: %v", err) } // Ensure that node is in state STARTING before the other node joins. @@ -719,19 +735,20 @@ func TestCluster_ResizeStates(t *testing.T) { // Expect an error by adding a node not in the topology. expectedError := "host is not in topology: node1" - err := tc.addNode() - if err == nil || err.Error() != expectedError { + if err := tc.addNode(); err == nil || err.Error() != expectedError { t.Errorf("did not receive expected error: %s", expectedError) } - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node2 := tc.Clusters[2] // Ensure that node comes up in state NORMAL. if node0.State() != ClusterStateNormal { t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State()) } // Close TestCluster. @@ -742,7 +759,9 @@ func TestCluster_ResizeStates(t *testing.T) { t.Run("Multiple nodes, with data", func(t *testing.T) { tc := NewClusterCluster(0) - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node0 := tc.Clusters[0] // Open TestCluster. @@ -752,10 +771,14 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Bit Data to node0. if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil { - t.Fatal(err) + t.Fatalf("creating field: %v", err) + } + if err := tc.SetBit("i", "f", 1, 101, nil); err != nil { + t.Fatalf("setting bit: %v", err) + } + if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil { + t.Fatalf("setting bit: %v", err) } - tc.SetBit("i", "f", 1, 101, nil) - tc.SetBit("i", "f", 1, 1300000, nil) // Before starting the resize, get the CheckSum to use for // comparison later. @@ -765,7 +788,9 @@ func TestCluster_ResizeStates(t *testing.T) { node0Checksum := node0Fragment.Checksum() // addNode needs to block until the resize process has completed. - tc.addNode() + if err := tc.addNode(); err != nil { + t.Fatalf("adding node: %v", err) + } node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. diff --git a/cmd/root.go b/cmd/root.go index 64b8b1ca9..3ff5c22e4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,11 +51,13 @@ Build Time: ` + pilosa.BuildTime + "\n", } // return "dry run" error if "dry-run" flag is set - if ret, err := cmd.Flags().GetBool("dry-run"); ret && err == nil { + ret, err := cmd.Flags().GetBool("dry-run") + if err != nil { + return fmt.Errorf("problem getting dry-run flag: %v", err) + } + if ret { if cmd.Parent() != nil { return fmt.Errorf("dry run") - } else if err != nil { - return fmt.Errorf("problem getting dry-run flag: %v", err) } } diff --git a/cmd/root_test.go b/cmd/root_test.go index 28bbea20d..6807aaf3c 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -187,10 +187,12 @@ bind = "127.0.0.1:10101" "127.0.0.1:10101", "127.0.0.1:10111", ]` - file.Write([]byte(config)) + if _, err := file.Write([]byte(config)); err != nil { + t.Fatalf("writing config file: %v", err) + } file.Close() _, err = ExecNewRootCommand(t, "server", "--config", file.Name()) - if err.Error() != "invalid option in configuration file: cluster.partitions" { + if err == nil || err.Error() != "invalid option in configuration file: cluster.partitions" { t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err) } } diff --git a/ctl/check.go b/ctl/check.go index b1389b6ee..0beee5bc5 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -68,7 +68,7 @@ func (cmd *CheckCommand) Run(_ context.Context) error { } // checkBitmapFile performs a consistency check on path for a roaring bitmap file. -func (cmd *CheckCommand) checkBitmapFile(path string) error { +func (cmd *CheckCommand) checkBitmapFile(path string) (err error) { // Open file handle. f, err := os.Open(path) if err != nil { @@ -86,8 +86,17 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { if err != nil { return errors.Wrap(err, "mmapping") } - defer syscall.Munmap(data) - + defer func() { + e := syscall.Munmap(data) + if e != nil { + fmt.Fprintf(cmd.Stderr, "WARNING: munmap failed: %v", e) + } + // don't overwrite another error with this, but also indicate + // this error. + if err == nil { + err = e + } + }() // Attach the mmap file to the bitmap. bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { diff --git a/ctl/check_test.go b/ctl/check_test.go index 1b2ebd6f6..e37d99358 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -40,7 +40,9 @@ func TestCheckCommand_RunCacheFile(t *testing.T) { err := cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.Contains(buf.String(), "ignoring cache file") { t.Fatalf("expect: ignoring cache file, actual: '%s'", err) @@ -59,7 +61,9 @@ func TestCheckCommand_RunSnapshot(t *testing.T) { err := cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.Contains(buf.String(), "ignoring snapshot file") { t.Fatalf("expect: ignoring snapshot file, actual: '%s'", err) @@ -71,7 +75,9 @@ func TestCheckCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("1234,1223")) + if _, err := file.Write([]byte("1234,1223")); err != nil { + t.Fatalf("writing to temp file: %v", err) + } file.Close() rder := []byte{} @@ -83,7 +89,9 @@ func TestCheckCommand_Run(t *testing.T) { err = cm.Run(context.Background()) w.Close() var buf bytes.Buffer - io.Copy(&buf, r) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("copy: %v", err) + } if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: reading roaring header:") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) diff --git a/ctl/config_test.go b/ctl/config_test.go index ca08c273b..b9a5dc0b5 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -33,13 +33,16 @@ func TestConfigCommand_Run(t *testing.T) { cm.Config = server.NewConfig() err := cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) - if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), ":10101") { + } + w.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: \n%s", buf.String()) } } diff --git a/ctl/export_test.go b/ctl/export_test.go index e3189efe3..e9f7db8a7 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -44,7 +44,9 @@ func TestExportCommand_Validation(t *testing.T) { } func TestExportCommand_Run(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -52,8 +54,16 @@ func TestExportCommand_Run(t *testing.T) { hostport := cmd.API.Node().URI.HostPort() cm.Host = hostport - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) + resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("making http request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) + if err != nil { + t.Fatalf("making http request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 0e59ce097..5a7e0db9f 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -42,7 +42,7 @@ func (cmd *GenerateConfigCommand) Run(_ context.Context) error { conf := server.NewConfig() ret, err := toml.Marshal(*conf) if err != nil { - return errors.Wrap(err, "unmarshaling default config") + return errors.Wrap(err, "unmarshalling default config") } fmt.Fprintf(cmd.Stdout, "%s\n", ret) return nil diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 26b531e8f..a431cd468 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -29,12 +29,16 @@ func TestGenerateConfigCommand_Run(t *testing.T) { r, w, _ := os.Pipe() cm := NewGenerateConfigCommand(stdin, w, os.Stderr) err := cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), ":10101") { + } + w.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import_test.go b/ctl/import_test.go index e701af1b2..dd4d4a9e6 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -58,13 +58,21 @@ func TestImportCommand_Basic(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } ctx := context.Background() if err != nil { t.Fatal(err) } - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -82,13 +90,18 @@ func TestImportCommand_Basic(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("1,2\n3,4\n5,6")) - ctx := context.Background() if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -110,17 +123,30 @@ func TestImportCommand_RunValue(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("1,2\n3,4\n5,6")) - ctx := context.Background() if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("http request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("http request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -136,17 +162,33 @@ func TestImportCommand_RunValue(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } ctx := context.Background() if err != nil { t.Fatal(err) } - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -165,17 +207,30 @@ func TestImportCommand_RunKeys(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("foo1,bar2\nfoo3,bar4\nfoo5,bar6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -192,7 +247,9 @@ func TestImportCommand_KeyReplication(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - + if err != nil { + t.Fatal(err) + } // create a large import file in order to test the // translateStoreBufferSize growth logic. keyBytes := []byte{} @@ -205,13 +262,14 @@ func TestImportCommand_KeyReplication(t *testing.T) { x := "fooEND,barEND" keyBytes = append(keyBytes, x...) - file.Write(keyBytes) - ctx := context.Background() + _, err = file.Write(keyBytes) if err != nil { - t.Fatal(err) + t.Fatalf("writing to tempfile: %v", err) } + ctx := context.Background() c := test.MustRunCluster(t, 2) + defer c.Close() cmd0 := c[0] cmd1 := c[1] @@ -220,8 +278,16 @@ func TestImportCommand_KeyReplication(t *testing.T) { cm.Host = host0 - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -255,17 +321,30 @@ func TestImportCommand_RunValueKeys(t *testing.T) { stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-key.csv") - file.Write([]byte("foo1,2\nfoo3,4\nfoo5,6")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("foo1,2\nfoo3,4\nfoo5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -277,7 +356,9 @@ func TestImportCommand_RunValueKeys(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -286,9 +367,12 @@ func TestImportCommand_InvalidFile(t *testing.T) { cm.Index = "i" cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") - file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("a,2\n3,5\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) } cm.Paths = []string{file.Name()} err = cm.Run(context.Background()) @@ -297,9 +381,12 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1,\n3,\n5,6")) if err != nil { - t.Fatal(err) + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,\n3,\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) } cm.Paths = []string{file.Name()} err = cm.Run(context.Background()) @@ -308,7 +395,10 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1,2,34343\n1,3,54565,\n5,6,565")) + if err != nil { + t.Fatal(err) + } + _, err = file.Write([]byte("1,2,34343\n1,3,54565,\n5,6,565")) if err != nil { t.Fatal(err) } @@ -319,7 +409,10 @@ func TestImportCommand_InvalidFile(t *testing.T) { } file, err = ioutil.TempFile("", "import1.csv") - file.Write([]byte("1\n3\n5")) + if err != nil { + t.Fatal(err) + } + _, err = file.Write([]byte("1\n3\n5")) if err != nil { t.Fatal(err) } @@ -351,22 +444,35 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { } func TestImportCommand_BugOverwriteValue(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import-value.csv") - file.Write([]byte("0,17\n")) - ctx := context.Background() if err != nil { t.Fatal(err) } + _, err = file.Write([]byte("0,17\n")) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -381,7 +487,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("0,16\n")) + _, err = file.Write([]byte("0,16\n")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -393,7 +502,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("0,19\n")) + _, err = file.Write([]byte("0,19\n")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -408,11 +520,21 @@ func TestImportCommand_RunBool(t *testing.T) { cm := NewImportCommand(stdin, stdout, stderr) ctx := context.Background() - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] cm.Host = cmd.API.Node().URI.HostPort() - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() + resp, err = http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + if err != nil { + t.Fatalf("posting request: %v", err) + } + resp.Body.Close() cm.Index = "i" cm.Field = "f" @@ -422,7 +544,10 @@ func TestImportCommand_RunBool(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("0,1\n1,2\n1,3")) + _, err = file.Write([]byte("0,1\n1,2\n1,3")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) @@ -437,8 +562,10 @@ func TestImportCommand_RunBool(t *testing.T) { if err != nil { t.Fatal(err) } - file.Write([]byte("0,1\n1,2\n1,3\n2,4")) - + _, err = file.Write([]byte("0,1\n1,2\n1,3\n2,4")) + if err != nil { + t.Fatalf("writing bytes to tempfile: %v", err) + } cm.Paths = []string{file.Name()} err = cm.Run(ctx) if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") { diff --git a/ctl/inspect.go b/ctl/inspect.go index c0676f8da..98222d0de 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -64,11 +64,15 @@ func (cmd *InspectCommand) Run(_ context.Context) error { if err != nil { return errors.Wrap(err, "mmapping") } - defer syscall.Munmap(data) - + defer func() { + err := syscall.Munmap(data) + if err != nil { + fmt.Fprintf(cmd.Stderr, "inspect command: munmap failed: %v", err) + } + }() // Attach the mmap file to the bitmap. t := time.Now() - fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") + fmt.Fprintf(cmd.Stderr, "unmarshalling bitmap...") bm := roaring.NewBitmap() if err := bm.UnmarshalBinary(data); err != nil { return errors.Wrap(err, "unmarshalling") diff --git a/ctl/inspect_test.go b/ctl/inspect_test.go index c7a48d400..bb87f894d 100644 --- a/ctl/inspect_test.go +++ b/ctl/inspect_test.go @@ -34,15 +34,24 @@ func TestInspectCommand_Run(t *testing.T) { if err != nil { t.Fatalf("Error creating tempfile: %s", err) } - file.Write([]byte("12358267538963")) + _, err = file.Write([]byte("12358267538963")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } file.Close() cm.Path = file.Name() err = cm.Run(context.Background()) + if err != nil && err.Error() != "unmarshalling: reading roaring header: did not find expected serialCookie in header" { + t.Fatalf("can't run command: %v", err) + } w.Close() var buf bytes.Buffer - io.Copy(&buf, r) - if !strings.Contains(buf.String(), "unmarshaling bitmap...") { + _, err = io.Copy(&buf, r) + if err != nil { + t.Fatalf("copying data: %v", err) + } + if !strings.Contains(buf.String(), "unmarshalling bitmap...") { t.Fatalf("Inspect doesn't work: %s", err) } diff --git a/diagnostics.go b/diagnostics.go index 1f9466772..c16cd9b0d 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -230,11 +230,11 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() { for _, index := range d.server.holder.Indexes() { numShards += index.AvailableShards().Count() - numIndexes += 1 + numIndexes++ for _, field := range index.Fields() { - numFields += 1 + numFields++ if field.Type() == FieldTypeInt { - bsiFieldCount += 1 + bsiFieldCount++ } if field.TimeQuantum() != "" { timeQuantumEnabled = true diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index 99c0a83ac..6ddc98b32 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -22,6 +22,8 @@ import ( "runtime" "strings" "testing" + + "github.com/pilosa/pilosa/logger" ) func TestDiagnosticsClient(t *testing.T) { @@ -112,19 +114,34 @@ func TestDiagnosticsVersion_Check(t *testing.T) { // Mock server. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(versionResponse{ + err := json.NewEncoder(w).Encode(versionResponse{ Version: "1.1.1", }) + if err != nil { + t.Fatalf("couldn't encode version response: %v", err) + } })) // Create a new client. d := newDiagnosticsCollector("localhost:10101") + logs := logger.NewCaptureLogger() + d.Logger = logs + version := "0.1.1" d.SetVersion(version) d.VersionURL = server.URL - d.CheckVersion() + err := d.CheckVersion() + if err != nil { + t.Fatalf("checking version: %v", err) + } + if len(logs.Prints) != 1 { + t.Fatalf("expected a version upgrade message") + } + if !strings.Contains(logs.Prints[0], "a newer version") { + t.Fatalf("expected version upgrade message, got '%s'", logs.Prints[0]) + } } func compareJSON(a, b []byte) (bool, error) { diff --git a/docs/administration.md b/docs/administration.md index 4bfa5470a..33fa73104 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,6 +56,11 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` +We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at: +* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) +* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) +* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) + ##### Importing Integer Values If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`. @@ -283,13 +288,13 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. - **GoRoutines:** Go routine count. - + You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. ### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. -The metrics configuration options are: +The metrics configuration options are: - [Host](../configuration/#metric-host): specify host that receives metric events - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics diff --git a/docs/api-reference.md b/docs/api-reference.md index 959c850d6..68dae46b5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -241,7 +241,7 @@ curl -XDELETE localhost:10101/index/user/field/language Returns the schema of all indexes in JSON. ``` request -curl -XGET localhost:10101/index +curl -XGET localhost:10101/schema ``` ``` response { @@ -276,6 +276,24 @@ curl -XGET localhost:10101/index } ``` +### Duplicate schema into empty Pilosa cluster + +`POST /schema` + +To duplicate one Pilosa cluster's schema to another, it's possible to +pass the output of `GET /schema` as the request body of `POST /schema` +and all the indexes and fields in the schema will be created in +Pilosa. As of this writing, the behavior of POSTing a schema to a +non-empty Pilosa cluster is undefined. These semantics will likely be +ironed out in a future version. + +``` request +# after (e.g.) curl -XGET localhost:10101/schema > schema.json +curl -XPOST localhost:10101/schema --data-binary @schema.json +``` + +Response: `204 No Content` + ### Get version `GET /version` diff --git a/docs/client-libraries.md b/docs/client-libraries.md index ac008def4..9492f7693 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -10,281 +10,9 @@ nav = [ ## Client Libraries -This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is best to avoid using the output of a hash or randomly distributed ids with Pilosa. +We have the following official client libraries. You can find more information in their repositories: +* [Go client repository](https://github.com/pilosa/go-pilosa) +* [Java client repository](https://github.com/pilosa/java-pilosa) +* [Python client repository](https://github.com/pilosa/python-pilosa) -### Go - -You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```go -package main - -import ( - "fmt" - - "github.com/pilosa/go-pilosa" -) - -func main() { - // We will just use the default client which assumes the server is at http://localhost:10101 - client := pilosa.DefaultClient() - - // Let's load the schema from the server. - // Note that, for this example the schema should be created beforehand - // and the stargazer data should be imported. - // See the Getting Started repository: https://github.com/pilosa/getting-started/ - schema, err := client.Schema() - if err != nil { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - // Error handling was left out for brevity in the rest of the code. - panic(err) - } - - // We need to refer to indexes and fields before we can use them in a query. - repository := schema.Index("repository") - stargazer := repository.Field("stargazer") - language := repository.Field("language") - - var response *pilosa.QueryResponse - - // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Row(14)) - fmt.Println("User 14 starred: ", response.Result().Row().Columns) - - // What are the top 5 languages in the sample data? - response, err = client.Query(language.TopN(5)) - languageIDs := []uint64{} - for _, item := range response.Result().CountItems() { - languageIDs = append(languageIDs, item.ID) - } - fmt.Println("Top 5 languages: ", languageIDs) - - // Which repositories were starred by both user 14 and 19: - response, _ = client.Query( - repository.Intersect( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19: - response, _ = client.Query( - repository.Union( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19 and were written in language 1: - response, _ = client.Query( - repository.Intersect( - repository.Union( - stargazer.Row(14), - stargazer.Row(19), - ), - language.Row(1))) - fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) - - // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.Set(99999, 77777)) -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514] -Top 5 languages: [5 1 4 9 13] -Both user 14 and 19 starred: [2 3 362 396 416 461 464 466 470 486] -User 14 or 19 starred: [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514] -User 14 or 19 starred, written in language 1: [1 2 362 368 382 386 416 426 435 456 461 483 500 503 504 514] -``` - -### Python - -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```python -from __future__ import print_function -from pilosa import Index, Client, PilosaError, TimeQuantum - -# We will just use the default client which assumes the server is at http://localhost:10101 -client = Client() - -# Let's load the schema from the server. -# Note that, for this example the schema should be created beforehand -# and the stargazer data should be imported. -# See the Getting Started repository: https://github.com/pilosa/getting-started/ - -# Let's create Index and Field objects, which will contain the settings -# for the corresponding indexes and fields. -try: - schema = client.schema() -except PilosaError as e: - # Most calls will raise an exception on errors. - # You should handle them appropriately. - # We will just terminate the program in this case. - raise SystemExit(e) - -# We need to refer to indexes and fields before we can use them in a query. -repository = schema.index("repository") -stargazer = repository.field("stargazer") -language = repository.field("language") - -# Which repositories did user 8 star: -repository_ids = client.query(stargazer.row(14)).result.row.columns -print("User 8 starred: ", repository_ids) - -# What are the top 5 languages in the sample data: -top_languages = client.query(language.topn(5)).result.count_items -print("Top 5 languages: ", [item.id for item in top_languages]) - -# Which repositories were starred by both user 14 and 19: -query = repository.intersect( - stargazer.row(14), - stargazer.row(19) -) -mutually_starred = client.query(query).result.row.columns -print("Both user 14 and 19 starred:", mutually_starred) - -# Which repositories were starred by user 14 or 19: -query = repository.union( - stargazer.row(14), - stargazer.row(19) -) -either_starred = client.query(query).result.row.columns -print("User 14 or 19 starred:", either_starred) - -# Which repositories were starred by user 14 or 19 and were written in language 1: -query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) -) -mutually_starred = client.query(query).result.row.columns -print("User 14 or 19 starred, written in language 1:", mutually_starred) - -# Set user 99999 as a stargazer for repository 77777 -client.query(stargazer.set(99999, 77777)) -``` - -Running the above program should produce output like this: -``` -('User 8 starred: ', [1L, 2L, 3L, 362L, 368L, 391L, 396L, 409L, 416L, 430L, 436L, 450L, 454L, 460L, 461L, 464L, 466L, 469L, 470L, 483L, 484L, 486L, 490L, 491L, 503L, 504L, 514L]) -('Top 5 languages: ', [5L, 1L, 4L, 9L, 13L]) -('Both user 14 and 19 starred:', [2L, 3L, 362L, 396L, 416L, 461L, 464L, 466L, 470L, 486L]) -('User 14 or 19 starred:', [1L, 2L, 3L, 361L, 362L, 368L, 376L, 377L, 378L, 382L, 386L, 388L, 391L, 396L, 398L, 400L, 409L, 411L, 412L, 416L, 426L, 428L, 430L, 435L, 436L, 450L, 452L, 453L, 454L, 456L, 460L, 461L, 464L, 465L, 466L, 469L, 470L, 483L, 484L, 486L, 487L, 489L, 490L, 491L, 500L, 503L, 504L, 505L, 512L, 514L]) -('User 14 or 19 starred, written in language 1:', [1L, 2L, 362L, 368L, 382L, 386L, 416L, 426L, 435L, 456L, 461L, 483L, 500L, 503L, 504L, 514L]) -``` - -### Java - -You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```java -import com.pilosa.client.*; -import com.pilosa.client.orm.*; -import com.pilosa.client.exceptions.PilosaException; - -import java.util.ArrayList; -import java.util.List; - -public class StarTrace { - public static void main(String[] args) { - // We will just use the default client which assumes the server is at http://localhost:10101 - PilosaClient client = PilosaClient.defaultClient(); - - // Let's load the schema from the server. - Schema schema; - try { - schema = client.readSchema(); - } - catch (PilosaException ex) { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - throw new RuntimeException(ex); - } - - // We need to refer to indexes and fields before we can use them in a query. - Index repository = schema.index("repository"); - Field stargazer = repository.field("stargazer"); - Field language = repository.field("language"); - - QueryResponse response; - QueryResult result; - PqlQuery query; - List repositoryIDs; - - // Which repositories did user 14 star: - response = client.query(stargazer.row(14)); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 starred: " + repositoryIDs); - - // What are the top 5 languages in the sample data: - response = client.query(language.topN(5)); - List top_languages = response.getResult().getCountItems(); - List languageIDs = new ArrayList(); - for (CountResultItem item : top_languages) { - languageIDs.add(item.getID()); - } - - System.out.println("Top Languages: " +languageIDs); - - // Which repositories were starred by both user 14 and 19: - query = repository.intersect( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("Both user 14 and 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19: - query = repository.union( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19 and were written in language 1: - query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); - - // Set user 99999 as a stargazer for repository 77777: - client.query(stargazer.set(99999, 77777)); - } -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514] -Top Languages: [5, 1, 4, 9, 13] -Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486] -User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514] -User 14 or 19 starred, written in language 1: [1, 2, 362, 368, 382, 386, 416, 426, 435, 456, 461, 483, 500, 503, 504, 514] -``` +Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients. diff --git a/docs/data-model.md b/docs/data-model.md index 1a0781ccc..7de27b575 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -165,6 +165,14 @@ Set(3, B=6) Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. + +###### BSI Deprecated Format + +The original implementation of BSI required a fixed bit depth when creating fields because the existence bit was written to the bit above the highest bit. The second version of BSI moves the existence bit to the beginning, adds a negative bit as the second bit, and shifts all remaining bits up by two. + +Pilosa automatically converts all old data to the new format on startup, however, this can cause issues when upgrading Pilosa and then reverting back to an old version. This documentation section exists as a record for anyone who experiences unusual behavior in BSI between versions. + + #### Time Time fields are similar to `set` fields, but in addition to row and column information, they also store a per-bit time value down to a defined granularity. The following example creates a `time` field called "event" which stores timestamp information down to a day granularity. @@ -186,7 +194,7 @@ Set(3, A=8, 2017-05-19T00:00) ``` ![time quantum field diagram](/img/docs/field-time-quantum.png) -*Time quantum fueld diagram* +*Time quantum field diagram* #### Mutex diff --git a/docs/getting-started.md b/docs/getting-started.md index 257e77628..c7dd54a8c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -63,6 +63,7 @@ curl localhost:10101/index/repository -X POST ``` response {"success":true} ``` +The index name must be 64 characters or less, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. Let's create the `stargazer` field which has user IDs of stargazers as its rows: ``` request @@ -74,7 +75,7 @@ curl localhost:10101/index/repository/field/stargazer \ {"success":true} ``` -Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. +Since our data contains time stamps which represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` field, which will contain IDs for programming languages: ``` request diff --git a/docs/installation.md b/docs/installation.md index 69de1376e..bebc5d53d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.3.0/pilosa-v1.3.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.2.0-darwin-amd64.tar.gz + tar xfz pilosa-v1.3.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.2.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.3.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -201,19 +201,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.3.0/pilosa-v1.3.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.2.0-linux-amd64.tar.gz + tar xfz pilosa-v1.3.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.2.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.3.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -230,7 +230,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -293,7 +293,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.2.0 + Version: v1.3.0 Build Time: 2018-05-14T22:14:01+0000 Usage: diff --git a/docs/query-language.md b/docs/query-language.md index 408a967bc..0064401cc 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -43,7 +43,7 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with a lowercase letter, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04) * `UINT` An unsigned integer (e.g. 42839) * `BOOL` A boolean value, `true` or `false` diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index d40db1fb9..a04ab929c 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -532,6 +532,8 @@ func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { CacheSize: o.CacheSize, Min: o.Min, Max: o.Max, + Base: o.Base, + BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), Keys: o.Keys, } @@ -800,6 +802,8 @@ func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) m.CacheSize = options.CacheSize m.Min = options.Min m.Max = options.Max + m.Base = options.Base + m.BitDepth = uint(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) m.Keys = options.Keys } diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go deleted file mode 100644 index c61f5c5b6..000000000 --- a/enterprise/b/btree.go +++ /dev/null @@ -1,953 +0,0 @@ -// This file is a modified redistribution of b (https://github.com/cznic/b), -// which is governed by the following license notice: -// -// Copyright (c) 2014 The b Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the names of the authors nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package b - -import ( - "io" - "sync" - - "github.com/pilosa/pilosa/roaring" -) - -const ( - // kx must be >= 2 - kx = 128 //TODO benchmark tune this number if using custom key/value type(s). - // kd must be >= 1 - kd = 128 //TODO benchmark tune this number if using custom key/value type(s). -) - -var ( - btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}} - btXPool = sync.Pool{New: func() interface{} { return &x{} }} -) - -type btTpool struct{ sync.Pool } - -func (p *btTpool) get(cmp Cmp) *tree { - x := p.Get().(*tree) - x.cmp = cmp - return x -} - -type btEpool struct{ sync.Pool } - -func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator { - x := p.Get().(*enumerator) - x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver - return x -} - -type ( - // Cmp compares a and b. Return value is: - // - // < 0 if a < b - // 0 if a == b - // > 0 if a > b - // - Cmp func(a, b uint64) int64 - - d struct { // data page - c int - d [2*kd + 1]de - n *d - p *d - } - - de struct { // d element - k uint64 - v *roaring.Container - } - - // enumerator captures the state of enumerating a tree. It is returned - // from the Seek* methods. The enumerator is aware of any mutations - // made to the tree in the process of enumerating it and automatically - // resumes the enumeration at the proper key, if possible. - // - // However, once an enumerator returns io.EOF to signal "no more - // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an enumerator is "sticky" (idempotent). - enumerator struct { - err error - hit bool - i int - k uint64 - q *d - t *tree - ver int64 - } - - // tree is a B+tree. - tree struct { - c int - cmp Cmp - first *d - last *d - r interface{} - ver int64 - } - - xe struct { // x element - ch interface{} - k uint64 - } - - x struct { // index page - c int - x [2*kx + 2]xe - } -) - -var ( // R/O zero values - zd d - zde de - ze enumerator - zk uint64 - zt tree - zx x - zxe xe -) - -func clr(q interface{}) { - switch x := q.(type) { - case *x: - for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn - clr(x.x[i].ch) - } - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } -} - -// -------------------------------------------------------------------------- x - -func newX(ch0 interface{}) *x { - r := btXPool.Get().(*x) - r.x[0].ch = ch0 - return r -} - -func (q *x) extract(i int) { - q.c-- - if i < q.c { - copy(q.x[i:], q.x[i+1:q.c+1]) - q.x[q.c].ch = q.x[q.c+1].ch - q.x[q.c].k = zk // GC - q.x[q.c+1] = zxe // GC - } -} - -func (q *x) insert(i int, k uint64, ch interface{}) *x { - c := q.c - if i < c { - q.x[c+1].ch = q.x[c].ch - copy(q.x[i+2:], q.x[i+1:c]) - q.x[i+1].k = q.x[i].k - } - c++ - q.c = c - q.x[i].k = k - q.x[i+1].ch = ch - return q -} - -func (q *x) siblings(i int) (l, r *d) { - if i >= 0 { - if i > 0 { - l = q.x[i-1].ch.(*d) - } - if i < q.c { - r = q.x[i+1].ch.(*d) - } - } - return l, r -} - -// -------------------------------------------------------------------------- d - -func (l *d) mvL(r *d, c int) { - copy(l.d[l.c:], r.d[:c]) - copy(r.d[:], r.d[c:r.c]) - // Zero out the de's here to prevent reading bad data - // and to avoid creating non-collectible (GC) references. - for i := 1; i < c; i++ { - r.d[r.c-i] = zde - } - l.c += c - r.c -= c -} - -func (l *d) mvR(r *d, c int) { - copy(r.d[c:], r.d[:r.c]) - copy(r.d[:c], l.d[l.c-c:]) - // Zero out the de's here to prevent reading bad data - // and to avoid creating non-collectible (GC) references. - for i := 1; i < c; i++ { - l.d[l.c-c+i] = zde - } - r.c += c - l.c -= c -} - -// ----------------------------------------------------------------------- Tree - -// treeNew returns a newly created, empty Tree. The compare function is used -// for key collation. -func treeNew(cmp Cmp) *tree { - return btTPool.get(cmp) -} - -// Clear removes all K/V pairs from the tree. -func (t *tree) Clear() { - if t.r == nil { - return - } - - clr(t.r) - t.c, t.first, t.last, t.r = 0, nil, nil, nil - t.ver++ -} - -// Close performs Clear and recycles t to a pool for possible later reuse. No -// references to t should exist or such references must not be used afterwards. -func (t *tree) Close() { - t.Clear() - *t = zt - btTPool.Put(t) -} - -func (t *tree) cat(p *x, q, r *d, pi int) { - t.ver++ - q.mvL(r, r.c) - if r.n != nil { - r.n.p = q - } else { - t.last = q - } - q.n = r.n - *r = zd - btDPool.Put(r) - if p.c > 1 { - p.extract(pi) - p.x[pi].ch = q - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -func (t *tree) catX(p, q, r *x, pi int) { - t.ver++ - q.x[q.c].k = p.x[pi].k - copy(q.x[q.c+1:], r.x[:r.c]) - q.c += r.c + 1 - q.x[q.c].ch = r.x[r.c].ch - *r = zx - btXPool.Put(r) - if p.c > 1 { - p.c-- - pc := p.c - if pi < pc { - p.x[pi].k = p.x[pi+1].k - copy(p.x[pi+1:], p.x[pi+2:pc+1]) - p.x[pc].ch = p.x[pc+1].ch - p.x[pc].k = zk // GC - p.x[pc+1].ch = nil // GC - } - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -// Delete removes the k's KV pair, if it exists, in which case Delete returns -// true. -func (t *tree) Delete(k uint64) (ok bool) { - pi := -1 - var p *x - q := t.r - if q == nil { - return false - } - - for { - var i int - i, ok = t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[pi].ch - continue - case *d: - t.extract(x, i) - if x.c >= kd { - return true - } - - if q != t.r { - t.underflow(p, x, pi) - } else if t.c == 0 { - t.Clear() - } - return true - } - } - - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - return false - } - } -} - -func (t *tree) extract(q *d, i int) { // (r *container) { - t.ver++ - //r = q.d[i].v // prepared for Extract - q.c-- - if i < q.c { - copy(q.d[i:], q.d[i+1:q.c+1]) - } - q.d[q.c] = zde // GC - t.c-- -} - -func (t *tree) find(q interface{}, k uint64) (i int, ok bool) { - var mk uint64 - l := 0 - switch x := q.(type) { - case *x: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.x[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - case *d: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.d[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - } - return l, false -} - -// First returns the first item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *tree) First() (k uint64, v *roaring.Container) { - if q := t.first; q != nil { - q := &q.d[0] - k, v = q.k, q.v - } - return k, v -} - -// Get returns the value associated with k and true if it exists. Otherwise Get -// returns (zero-value, false). -func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) { - q := t.r - if q == nil { - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return x.d[i].v, true - } - } - switch x := q.(type) { - case *x: - q = x.x[i].ch - default: - return - } - } -} - -func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { - t.ver++ - c := q.c - if i < c { - copy(q.d[i+1:], q.d[i:c]) - } - c++ - q.c = c - q.d[i].k, q.d[i].v = k, v - t.c++ - return q -} - -// Last returns the last item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *tree) Last() (k uint64, v *roaring.Container) { - if q := t.last; q != nil { - q := &q.d[q.c-1] - k, v = q.k, q.v - } - return k, v -} - -// Len returns the number of items in the tree. -func (t *tree) Len() int { - return t.c -} - -func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { - t.ver++ - l, r := p.siblings(pi) - - // s is the number of items to shift out of the full data container to - // allow for the new data item. This logic shifts by half the available - // space plus one. In the case where the new item is to be inserted within - // the calculated shift space, then s is reduced to include only the - // data items up to the index of the new data item. - if l != nil && l.c < 2*kd && i != 0 { - s := (2*kd-l.c)/2 + 1 // half plus one - //s := 2*kd - l.c // all available - if i < s { - s = i - } - l.mvL(q, s) - t.insert(q, i-s, k, v) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && r.c < 2*kd { - if i < 2*kd { - s := (2*kd-r.c)/2 + 1 // half plus one - //s := 2*kd - r.c // all available - if 2*kd-i < s { - s = 2*kd - i - } - q.mvR(r, s) - t.insert(q, i, k, v) - p.x[pi].k = r.d[0].k - return - } - - t.insert(r, 0, k, v) - p.x[pi].k = k - return - } - - t.split(p, q, pi, i, k, v) -} - -// Seek returns an Enumerator positioned on an item such that k >= item's key. -// ok reports if k == item.key The Enumerator's position is possibly after the -// last item in the tree. -func (t *tree) Seek(k uint64) (e *enumerator, ok bool) { - q := t.r - if q == nil { - e = btEPool.get(nil, false, 0, k, nil, t, t.ver) - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), true - } - } - - switch x := q.(type) { - case *x: - q = x.x[i].ch - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), false - } - } -} - -// SeekFirst returns an enumerator positioned on the first KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *tree) SeekFirst() (e *enumerator, err error) { - q := t.first - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil -} - -// SeekLast returns an enumerator positioned on the last KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *tree) SeekLast() (e *enumerator, err error) { - q := t.last - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil -} - -// Set sets the value associated with k. -func (t *tree) Set(k uint64, v *roaring.Container) { - //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) - //defer func() { - // dbg("--- POST\n%s\n====\n", t.dump()) - //}() - - pi := -1 - var p *x - q := t.r - if q == nil { - z := t.insert(btDPool.Get().(*d), 0, k, v) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - i++ - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - continue - case *d: - x.d[i].v = v - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - switch { - case x.c < 2*kd: - t.insert(x, i, k, v) - default: - t.overflow(p, x, pi, i, k, v) - } - return - } - } -} - -// Put combines Get and Set in a more efficient way where the tree is walked -// only once. The upd(ater) receives (old-value, true) if a KV pair for k -// exists or (zero-value, false) otherwise. It can then return a (new-value, -// true) to create or overwrite the existing value in the KV pair, or -// (whatever, false) if it decides not to create or not to update the value of -// the KV pair. -// -// tree.Set(k, v) call conceptually equals calling -// -// tree.Put(k, func(uint64, bool){ return v, true }) -// -// modulo the differing return values. -func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { - pi := -1 - var p *x - q := t.r - var newV *roaring.Container - if q == nil { - // new KV pair in empty tree - newV, written = upd(newV, false) - if !written { - return - } - - z := t.insert(btDPool.Get().(*d), 0, k, newV) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - i++ - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - continue - case *d: - oldV = x.d[i].v - newV, written = upd(oldV, true) - if !written { - return - } - - x.d[i].v = newV - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: // new KV pair - newV, written = upd(newV, false) - if !written { - return - } - - switch { - case x.c < 2*kd: - t.insert(x, i, k, newV) - default: - t.overflow(p, x, pi, i, k, newV) - } - return - } - } -} - -func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { - t.ver++ - r := btDPool.Get().(*d) - if q.n != nil { - r.n = q.n - r.n.p = r - } else { - t.last = r - } - q.n = r - r.p = q - - copy(r.d[:], q.d[kd:2*kd]) - for i := range q.d[kd:] { - q.d[kd+i] = zde - } - q.c = kd - r.c = kd - var done bool - if i > kd { - done = true - t.insert(r, i-kd, k, v) - } - if pi >= 0 { - p.insert(pi, r.d[0].k, r) - } else { - t.r = newX(q).insert(0, r.d[0].k, r) - } - if done { - return - } - - t.insert(q, i, k, v) -} - -func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - r := btXPool.Get().(*x) - copy(r.x[:], q.x[kx+1:]) - q.c = kx - r.c = kx - if pi >= 0 { - p.insert(pi, q.x[kx].k, r) - } else { - t.r = newX(q).insert(0, q.x[kx].k, r) - } - - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - if i > kx { - q = r - i -= kx + 1 - } - - return q, i -} - -func (t *tree) underflow(p *x, q *d, pi int) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c+q.c >= 2*kd { - l.mvR(q, 1) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && q.c+r.c >= 2*kd { - q.mvL(r, 1) - p.x[pi].k = r.d[0].k - r.d[r.c] = zde // GC - return - } - - if l != nil { - t.cat(p, l, q, pi-1) - return - } - - t.cat(p, q, r, pi) -} - -func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - var l, r *x - - if pi >= 0 { - if pi > 0 { - l = p.x[pi-1].ch.(*x) - } - if pi < p.c { - r = p.x[pi+1].ch.(*x) - } - } - - if l != nil && l.c > kx { - q.x[q.c+1].ch = q.x[q.c].ch - copy(q.x[1:], q.x[:q.c]) - q.x[0].ch = l.x[l.c].ch - q.x[0].k = p.x[pi-1].k - q.c++ - i++ - l.c-- - p.x[pi-1].k = l.x[l.c].k - return q, i - } - - if r != nil && r.c > kx { - q.x[q.c].k = p.x[pi].k - q.c++ - q.x[q.c].ch = r.x[0].ch - p.x[pi].k = r.x[0].k - copy(r.x[:], r.x[1:r.c]) - r.c-- - rc := r.c - r.x[rc].ch = r.x[rc+1].ch - r.x[rc].k = zk - r.x[rc+1].ch = nil - return q, i - } - - if l != nil { - i += l.c + 1 - t.catX(p, l, q, pi-1) - q = l - return q, i - } - - t.catX(p, q, r, pi) - return q, i -} - -// ----------------------------------------------------------------- Enumerator - -// Close recycles e to a pool for possible later reuse. No references to e -// should exist or such references must not be used afterwards. -func (e *enumerator) Close() { - *e = ze - btEPool.Put(e) -} - -// Next returns the currently enumerated item, if it exists and moves to the -// next item in the key collation order. If there is no item to return, err == -// io.EOF is returned. -func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) { - if err = e.err; err != nil { - return 0, nil, err - } - - if e.ver != e.t.ver { - f, _ := e.t.Seek(e.k) - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return 0, nil, err - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return 0, nil, err - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, true - e.next() - return k, v, nil -} - -func (e *enumerator) next() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i < e.q.c-1: - e.i++ - default: - if e.q, e.i = e.q.n, 0; e.q == nil { - e.err = io.EOF - } - } - return e.err -} - -// Prev returns the currently enumerated item, if it exists and moves to the -// previous item in the key collation order. If there is no item to return, err -// == io.EOF is returned. -func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) { - if err = e.err; err != nil { - return 0, nil, err - } - - if e.ver != e.t.ver { - f, _ := e.t.Seek(e.k) - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return 0, nil, err - } - - if !e.hit { - // move to previous because Seek overshoots if there's no hit - if err = e.prev(); err != nil { - return 0, nil, err - } - } - - if e.i >= e.q.c { - if err = e.prev(); err != nil { - return 0, nil, err - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, true - e.prev() - return k, v, err -} - -func (e *enumerator) prev() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i > 0: - e.i-- - default: - if e.q = e.q.p; e.q == nil { - e.err = io.EOF - break - } - - e.i = e.q.c - 1 - } - return e.err -} diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go deleted file mode 100644 index db5c9946d..000000000 --- a/enterprise/b/containers_btree.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright (c) 2018 Pilosa Corp. All rights reserved. -// -// This file is part of Pilosa Enterprise Edition. -// -// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Pilosa Enterprise Edition is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with Pilosa Enterprise Edition. If not, see . - -package b - -import ( - "io" - - "github.com/pilosa/pilosa/roaring" -) - -func cmp(a, b uint64) int64 { - return int64(a - b) -} - -type bTreeContainers struct { - tree *tree - - lastKey uint64 - lastContainer *roaring.Container -} - -func newBTreeContainers() *bTreeContainers { - return &bTreeContainers{ - tree: treeNew(cmp), - } -} - -func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { - b := &roaring.Bitmap{ - Containers: newBTreeContainers(), - } - b.Add(a...) - return b -} - -func (btc *bTreeContainers) Get(key uint64) *roaring.Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - var c *roaring.Container - el, ok := btc.tree.Get(key) - if ok { - c = el - btc.lastKey = key - btc.lastContainer = c - } - return c -} - -func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) { - // If a mapped container is added to the tree, reset the - // lastContainer cache so that the cache is not pointing - // at a read-only mmap. - if c.Mapped() { - btc.lastContainer = nil - } - btc.tree.Set(key, c) -} - -func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) { - // update the existing container - if exists { - oldV.Update(u.containerType, u.n, u.mapped) - return oldV, false - } - cont := roaring.NewContainer() - cont.Update(u.containerType, u.n, u.mapped) - return cont, true -} - -// this struct is added to prevent the closure locals from being escaped out to the heap -type updater struct { - key uint64 - n int32 - containerType byte - mapped bool -} - -func (btc *bTreeContainers) PutContainerValues(key uint64, typ byte, n int, mapped bool) { - a := updater{key, int32(n), typ, mapped} - btc.tree.Put(key, a.update) -} - -func (btc *bTreeContainers) Remove(key uint64) { - btc.tree.Delete(key) -} - -func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container { - // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { - return btc.lastContainer - } - - btc.lastKey = key - v, ok := btc.tree.Get(key) - if !ok { - cont := roaring.NewContainerArray(nil) - btc.tree.Set(key, cont) - btc.lastContainer = cont - return cont - } - - btc.lastContainer = v - return btc.lastContainer -} - -func (btc *bTreeContainers) Count() (n uint64) { - e, _ := btc.tree.Seek(0) - _, c, err := e.Next() - for err != io.EOF { - n += uint64(c.N()) - _, c, err = e.Next() - } - return n -} - -func (btc *bTreeContainers) Clone() roaring.Containers { - nbtc := newBTreeContainers() - - itr, err := btc.tree.SeekFirst() - if err == io.EOF { - return nbtc - } - for { - k, v, err := itr.Next() - if err == io.EOF { - break - } - nbtc.tree.Set(k, v.Clone()) - } - return nbtc -} - -func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) { - if btc.tree.Len() == 0 { - return 0, nil - } - k, v := btc.tree.Last() - return k, v -} - -func (btc *bTreeContainers) Size() int { - return btc.tree.Len() -} - -func (btc *bTreeContainers) Reset() { - btc.tree = treeNew(cmp) - btc.lastKey = 0 - btc.lastContainer = nil -} - -func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { - e, ok := btc.tree.Seek(key) - if ok { - found = true - } - - return &btcIterator{ - e: e, - }, found -} - -func (btc *bTreeContainers) Repair() { - e, _ := btc.tree.Seek(0) - _, c, err := e.Next() - for err != io.EOF { - c.Repair() - _, c, err = e.Next() - } -} - -type btcIterator struct { - e *enumerator - key uint64 - val *roaring.Container -} - -func (i *btcIterator) Next() bool { - - k, v, err := i.e.Next() - if err == io.EOF { - return false - } - i.key = k - i.val = v - return true -} - -func (i *btcIterator) Value() (uint64, *roaring.Container) { - if i.val == nil { - return 0, nil - } - return i.key, i.val -} diff --git a/enterprise/enterprise.go b/enterprise/enterprise.go index a75223801..f3a897551 100644 --- a/enterprise/enterprise.go +++ b/enterprise/enterprise.go @@ -21,13 +21,3 @@ // "ENTERPRISE=1 make install". These features were dual-licensed separately // from Pilosa community edition under the AGPL and Pilosa's commercial license. package enterprise - -import ( - "github.com/pilosa/pilosa/enterprise/b" - "github.com/pilosa/pilosa/roaring" -) - -func init() { // nolint: gochecknoinits - // Replace Bitmap constructor with B+Tree implementation - roaring.NewFileBitmap = b.NewBTreeBitmap -} diff --git a/event.go b/event.go index 0d5e59e99..b27bd1bf6 100644 --- a/event.go +++ b/event.go @@ -17,6 +17,7 @@ package pilosa // NodeEventType are the types of node events. type NodeEventType int +// Constant node event types. const ( NodeJoin NodeEventType = iota NodeLeave diff --git a/executor.go b/executor.go index b345d434b..607840a42 100644 --- a/executor.go +++ b/executor.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/shardwidth" "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -596,12 +597,12 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq return ValCount{}, nil } - vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ - Val: int64(vsum) + (int64(vcount) * bsig.Min), + Val: int64(vsum) + (int64(vcount) * bsig.Base), Count: int64(vcount), }, nil } @@ -637,12 +638,12 @@ func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmin) + bsig.Min, + Val: int64(fmin) + bsig.Base, Count: int64(fcount), }, nil } @@ -678,12 +679,12 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmax) + bsig.Min, + Val: int64(fmax) + bsig.Base, Count: int64(fcount), }, nil } @@ -765,11 +766,14 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") defer span.Finish() - field, _ := c.Args["_field"].(string) + fieldName, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNShard: %v", err) + } else if f := e.Holder.Field(index, fieldName); f != nil && f.Type() == FieldTypeInt { + return nil, fmt.Errorf("cannot compute TopN() on integer field: %q", fieldName) } + attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { @@ -798,13 +802,15 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca } // Set default field. - if field == "" { - field = defaultField + if fieldName == "" { + fieldName = defaultField } - f := e.Holder.fragment(index, field, viewStandard, shard) + f := e.Holder.fragment(index, fieldName, viewStandard, shard) if f == nil { return nil, nil + } else if f.CacheType == CacheTypeNone { + return nil, fmt.Errorf("cannot compute TopN(), field has no cache: %q", fieldName) } if minThreshold == 0 { @@ -986,6 +992,8 @@ type FieldRow struct { RowKey string `json:"rowKey,omitempty"` } +// MarshalJSON marshals FieldRow to JSON such that +// either a Key or an ID is included. func (fr FieldRow) MarshalJSON() ([]byte, error) { if fr.RowKey != "" { return json.Marshal(struct { @@ -1005,10 +1013,12 @@ func (fr FieldRow) MarshalJSON() ([]byte, error) { }) } +// String is the FieldRow stringer. func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } +// GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` @@ -1047,6 +1057,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { return ret } +// Compare is used in ordering two GroupCount objects. func (g GroupCount) Compare(o GroupCount) int { for i := range g.Group { if g.Group[i].RowID < o.Group[i].RowID { @@ -1157,7 +1168,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s // views contains the list of views to inspect (and merge) // in order to represent `Rows` for the field. - var views []string = []string{viewStandard} + var views = []string{viewStandard} // Handle `time` fields. if f.Type() == FieldTypeTime { @@ -1234,7 +1245,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - colShard := columnID >> shardWidthExponent + colShard := columnID >> shardwidth.Exponent if colShard != shard { return rowIDs, nil } @@ -1399,10 +1410,9 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return NewRow(), nil } - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } else if cond.Op == pql.BETWEEN { - predicates, err := cond.IntSliceValue() if err != nil { return nil, errors.Wrap(err, "getting condition value") @@ -1437,10 +1447,10 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // If the query is asking for the entire valid range, just return // the not-null bitmap for the bsiGroup. if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } - return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth, baseValueMin, baseValueMax) } else { @@ -1470,16 +1480,16 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.notNull(bsig.BitDepth()) + return frag.notNull() } f.Stats.Count("range:bsigroup", 1, 1.0) - return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) + return frag.rangeOp(cond.Op, bsig.BitDepth, baseValue) } } @@ -1705,11 +1715,11 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -1981,11 +1991,11 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -2016,11 +2026,11 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq } // Forward call to remote node otherwise. - if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil); err != nil { + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil) + if err != nil { return false, err - } else { - ret = res[0].(bool) } + ret = res[0].(bool) } return ret, nil } @@ -2530,6 +2540,16 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e } } + if filter, ok, err := c.CallArg("filter"); ok { + if err != nil { + return errors.Wrap(err, "getting filter call") + } + err = e.translateCall(index, idx, filter) + if err != nil { + return errors.Wrap(err, "translating filter call") + } + } + prev, ok := c.Args["previous"] if !ok { return nil // nothing else to be translated @@ -2608,7 +2628,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { - return nil, ErrFieldNotFound + return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { other := make([]Pair, len(result)) @@ -2880,7 +2900,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde return nil, errors.Wrap(err, "getting previous") } else if hasPrev && !ignorePrev { if i == len(children)-1 { - prev += 1 + prev++ } gbi.rowIters[i].Seek(prev) } diff --git a/executor_internal_test.go b/executor_internal_test.go index 4177968d5..ffb091d49 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/executor_test.go b/executor_test.go index 06133738c..3565c40fd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "io/ioutil" + "math" "math/rand" "reflect" "strconv" @@ -508,7 +509,9 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} hldr.SetBit("i", "f", 1, 0) @@ -571,7 +574,9 @@ func TestExecutor_Execute_Set(t *testing.T) { }) t.Run("RowKeyColumnKey", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) @@ -765,7 +770,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) @@ -802,7 +807,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1053,6 +1058,65 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(diff) } }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set data on the "f" field. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(0, f=1) + `}); err != nil { + t.Fatal(err) + } else if err := c[0].RecalculateCaches(); err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + // Attempt to query the "g" field. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(g, n=2)`}); err == nil || err.Error() != `executing: field "g" not found` { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("ErrBSIField", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create BSI "f" field. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("ErrCacheNone", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0)); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(0, f=1) + `}); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { + t.Fatalf("unexpected error: %v", err) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -1210,7 +1274,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { t.Fatal(err) } @@ -1258,32 +1322,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } } }) - - t.Run("Max", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 60, cnt: 1}, - {filter: `Row(x=0)`, exp: 60, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, - } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Max(field=f)` - } else { - pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } - } - }) }) t.Run("ColumnKey", func(t *testing.T) { @@ -1300,7 +1338,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1110, 1000)); err != nil { t.Fatal(err) } @@ -1393,15 +1431,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1451,15 +1489,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } @@ -1853,19 +1891,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-900, 1000)); err != nil { t.Fatal(err) } @@ -1889,8 +1927,8 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) { + t.Fatalf("Query().Row.Columns=%#v, expected %#v", got, exp) } }) @@ -1899,20 +1937,20 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -1927,8 +1965,8 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}; !reflect.DeepEqual(got, exp) { + t.Fatalf("unexpected result: got=%v, exp=%v", got, exp) } }) @@ -1936,7 +1974,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -1944,7 +1982,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) @@ -1964,7 +2002,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { {q: `Row(1000 < other <= 1000)`, exp: false}, {q: `Row(1000 < other < 2000)`, exp: false}, - {q: `Row(1000 <= other < 2000)`, exp: true}, + {q: `Row(1000 <= other < 20000)`, exp: true}, {q: `Row(1000 <= other <= 2000)`, exp: true}, {q: `Row(1000 < other <= 2000)`, exp: false}, } @@ -1977,7 +2015,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result for query: %s", test.q) + t.Fatalf("unexpected result for query: %s (%#v)", test.q, result.Results[0].(*pilosa.Row).Columns()) } }) } @@ -2018,7 +2056,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -200)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2047,19 +2085,19 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { t.Fatal(err) } @@ -2184,7 +2222,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -200)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -1200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2237,11 +2275,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Remote SetBit", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { - t.Fatalf("quuerying remote: %v", err) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil { + t.Fatalf("querying remote: %v", err) } - if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } }) @@ -2252,11 +2290,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil { t.Fatalf("quuerying remote: %v", err) } - if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) { + if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) { t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns()) } }) @@ -2817,7 +2855,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100)) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -3239,7 +3277,56 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { } }) } +} +func TestExecutor_GroupByStrings(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "generals", pilosa.OptFieldKeys()) + + req := &pilosa.ImportRequest{ + Index: "istring", + Field: "generals", + Shard: 0, + RowKeys: []string{"r1", "r2", "r1", "r2", "r1", "r2", "r1", "r2", "r1", "r2"}, + ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"}, + } + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatalf("importing: %v", err) + } + + tests := []struct { + query string + expected []pilosa.GroupCount + }{ + { + query: "GroupBy(Rows(generals))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5}, + }, + }, + { + query: "GroupBy(Rows(generals), filter=Row(generals=r2))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5}, + }, + }, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "istring", + Query: tst.query, + }) + if err != nil { + t.Fatalf("got an error %v", err) + } + results := r.Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, tst.expected, results) + }) + } } func TestExecutor_Execute_Rows_Keys(t *testing.T) { diff --git a/field.go b/field.go index bd4dd1aca..3e71af589 100644 --- a/field.go +++ b/field.go @@ -16,6 +16,7 @@ package pilosa import ( "bufio" + "context" "encoding/json" "fmt" "io/ioutil" @@ -32,6 +33,7 @@ import ( "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -58,10 +60,6 @@ const ( FieldTypeBool = "bool" ) -func defaultShardValidator(shard uint64) bool { - return true -} - // Field represents a container for views. type Field struct { mu sync.RWMutex @@ -84,7 +82,6 @@ type Field struct { // Shards with data on any node in the cluster, according to this node. remoteAvailableShards *roaring.Bitmap - shardValidator func(uint64) bool logger logger.Logger } @@ -92,6 +89,8 @@ type Field struct { // FieldOption is a functional option type for pilosa.fieldOptions. type FieldOption func(fo *FieldOptions) error +// OptFieldKeys is a functional option on FieldOptions +// used to specify whether keys are used for this field. func OptFieldKeys() FieldOption { return func(fo *FieldOptions) error { fo.Keys = true @@ -99,6 +98,8 @@ func OptFieldKeys() FieldOption { } } +// OptFieldTypeDefault is a functional option on FieldOptions +// used to set the field type and cache setting to the default values. func OptFieldTypeDefault() FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -111,6 +112,9 @@ func OptFieldTypeDefault() FieldOption { } } +// OptFieldTypeSet is a functional option on FieldOptions +// used to specify the field as being type `set` and to +// provide any respective configuration values. func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -123,13 +127,16 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { } } +// OptFieldTypeInt is a functional option on FieldOptions +// used to specify the field as being type `int` and to +// provide any respective configuration values. func OptFieldTypeInt(min, max int64) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) } if min > max { - return ErrInvalidBSIGroupRange + return errors.New("int field min cannot be greater than max") } fo.Type = FieldTypeInt fo.Min = min @@ -138,7 +145,9 @@ func OptFieldTypeInt(min, max int64) FieldOption { } } -// OptFieldTypeTime sets the field type to time. +// OptFieldTypeTime is a functional option on FieldOptions +// used to specify the field as being type `time` and to +// provide any respective configuration values. // Pass true to skip creation of the standard view. func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { return func(fo *FieldOptions) error { @@ -155,6 +164,9 @@ func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { } } +// OptFieldTypeMutex is a functional option on FieldOptions +// used to specify the field as being type `mutex` and to +// provide any respective configuration values. func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -167,6 +179,9 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { } } +// OptFieldTypeBool is a functional option on FieldOptions +// used to specify the field as being type `bool` and to +// provide any respective configuration values. func OptFieldTypeBool() FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -212,8 +227,7 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { remoteAvailableShards: roaring.NewBitmap(), - shardValidator: defaultShardValidator, - logger: logger.NopLogger, + logger: logger.NopLogger, } return f, nil } @@ -367,27 +381,33 @@ func (f *Field) Options() FieldOptions { func (f *Field) Open() error { if err := func() error { // Ensure the field's path exists. + f.logger.Debugf("ensure field path exists: %s", f.path) if err := os.MkdirAll(f.path, 0777); err != nil { return errors.Wrap(err, "creating field dir") } + f.logger.Debugf("load meta file for index/field: %s/%s", f.index, f.name) if err := f.loadMeta(); err != nil { return errors.Wrap(err, "loading meta") } + f.logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name) if err := f.loadAvailableShards(); err != nil { return errors.Wrap(err, "loading available shards") } // Apply the field options loaded from meta. + f.logger.Debugf("apply options for index/field: %s/%s", f.index, f.name) if err := f.applyOptions(f.options); err != nil { return errors.Wrap(err, "applying options") } + f.logger.Debugf("open views for index/field: %s/%s", f.index, f.name) if err := f.openViews(); err != nil { return errors.Wrap(err, "opening views") } + f.logger.Debugf("open row attribute store for index/field: %s/%s", f.index, f.name) if err := f.rowAttrStore.Open(); err != nil { return errors.Wrap(err, "opening attrstore") } @@ -398,6 +418,7 @@ func (f *Field) Open() error { return err } + f.logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name) return nil } @@ -422,11 +443,29 @@ func (f *Field) openViews() error { } name := filepath.Base(fi.Name()) + f.logger.Debugf("open index/field/view: %s/%s/%s", f.index, f.name, fi.Name()) view := f.newView(f.viewPath(name), name) if err := view.open(); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } + + // Automatically upgrade BSI v1 fragments if they exist & reopen view. + if bsig := f.bsiGroup(f.name); bsig != nil { + if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil { + return errors.Wrap(err, "upgrade view bsi v2") + } else if ok { + if err := view.close(); err != nil { + return errors.Wrap(err, "closing upgraded view") + } + view = f.newView(f.viewPath(name), name) + if err := view.open(); err != nil { + return fmt.Errorf("re-opening view: view=%s, err=%s", view.name, err) + } + } + } + view.rowAttrStore = f.rowAttrStore + f.logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name) f.viewMap[view.name] = view } @@ -449,12 +488,23 @@ func (f *Field) loadMeta() error { } } + // Initialize "base" to "min" when upgrading from v1 BSI format. + if pb.BitDepth == 0 { + pb.Base = pb.Min + pb.BitDepth = uint64(bitDepthInt64(pb.Max - pb.Min)) + if pb.BitDepth == 0 { + pb.BitDepth = 1 + } + } + // Copy metadata fields. f.options.Type = pb.Type f.options.CacheType = pb.CacheType f.options.CacheSize = pb.CacheSize f.options.Min = pb.Min f.options.Max = pb.Max + f.options.Base = pb.Base + f.options.BitDepth = uint(pb.BitDepth) f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) f.options.Keys = pb.Keys f.options.NoStandardView = pb.NoStandardView @@ -500,6 +550,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { } f.options.Min = 0 f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.TimeQuantum = "" f.options.Keys = opt.Keys case FieldTypeInt: @@ -508,15 +560,19 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = opt.Min f.options.Max = opt.Max + f.options.Base = opt.Base + f.options.BitDepth = opt.BitDepth f.options.TimeQuantum = "" f.options.Keys = opt.Keys // Create new bsiGroup. bsig := &bsiGroup{ - Name: f.name, - Type: bsiGroupTypeInt, - Min: opt.Min, - Max: opt.Max, + Name: f.name, + Type: bsiGroupTypeInt, + Min: opt.Min, + Max: opt.Max, + Base: opt.Base, + BitDepth: opt.BitDepth, } // Validate bsiGroup. if err := bsig.validate(); err != nil { @@ -531,6 +587,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.Keys = opt.Keys f.options.NoStandardView = opt.NoStandardView // Set the time quantum. @@ -544,6 +602,8 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 + f.options.Base = 0 + f.options.BitDepth = 0 f.options.TimeQuantum = "" f.options.Keys = false default: @@ -612,7 +672,9 @@ func (f *Field) createBSIGroup(bsig *bsiGroup) error { if err := f.addBSIGroup(bsig); err != nil { return err } - f.saveMeta() + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving") + } return nil } @@ -760,7 +822,6 @@ func (f *Field) newView(path, name string) *view { view.rowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster - view.shardValidator = f.shardValidator return view } @@ -941,18 +1002,18 @@ func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { return 0, false, nil } - v, exists, err := view.value(columnID, bsig.BitDepth()) + v, exists, err := view.value(columnID, bsig.BitDepth) if err != nil { return 0, false, err } else if !exists { return 0, false, nil } - return int64(v) + bsig.Min, true, nil + return int64(v) + bsig.Base, true, nil } // SetValue sets a field value for a column. func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { - // Fetch bsiGroup and validate value. + // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound @@ -962,16 +1023,37 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) return false, ErrBSIGroupValueTooHigh } + // Determine base value to store. + baseValue := int64(value - bsig.Base) + requiredBitDepth := bitDepthInt64(baseValue) + + // Increase bit depth value if the unsigned value is greater. + if requiredBitDepth > bsig.BitDepth { + if err := func() error { + f.mu.Lock() + defer f.mu.Unlock() + + uvalue := uint64(baseValue) + if value < 0 { + uvalue = uint64(-baseValue) + } + bitDepth := bitDepth(uvalue) + + bsig.BitDepth = bitDepth + f.options.BitDepth = bitDepth + return f.saveMeta() + }(); err != nil { + return false, errors.Wrap(err, "increasing bsi max") + } + } + // Fetch target view. view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name) if err != nil { return false, errors.Wrap(err, "creating view") } - // Determine base value to store. - baseValue := uint64(value - bsig.Min) - - return view.setValue(columnID, bsig.BitDepth(), baseValue) + return view.setValue(columnID, bsig.BitDepth, baseValue) } // Sum returns the sum and count for a field. @@ -987,11 +1069,11 @@ func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { return 0, 0, nil } - vsum, vcount, err := view.sum(filter, bsig.BitDepth()) + vsum, vcount, err := view.sum(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil + return int64(vsum) + (int64(vcount) * bsig.Base), int64(vcount), nil } // Min returns the min for a field. @@ -1007,11 +1089,11 @@ func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { return 0, 0, nil } - vmin, vcount, err := view.min(filter, bsig.BitDepth()) + vmin, vcount, err := view.min(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vmin) + bsig.Min, int64(vcount), nil + return int64(vmin) + bsig.Base, int64(vcount), nil } // Max returns the max for a field. @@ -1027,13 +1109,14 @@ func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { return 0, 0, nil } - vmax, vcount, err := view.max(filter, bsig.BitDepth()) + vmax, vcount, err := view.max(filter, bsig.BitDepth) if err != nil { return 0, 0, err } - return int64(vmax) + bsig.Min, int64(vcount), nil + return int64(vmax) + bsig.Base, int64(vcount), nil } +// Range performs a conditional operation on Field. func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { // Retrieve and validate bsiGroup. bsig := f.bsiGroup(name) @@ -1054,7 +1137,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) return NewRow(), nil } - return view.rangeOp(op, bsig.BitDepth(), baseValue) + return view.rangeOp(op, bsig.BitDepth, baseValue) } // Import bulk imports data. @@ -1147,6 +1230,36 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return errors.Wrap(ErrBSIGroupNotFound, f.name) } + // Find the lowest/highest values. + var min, max int64 + for i, value := range values { + if i == 0 || value < min { + min = value + } + if i == 0 || value > max { + max = value + } + } + + // Determine the highest bit depth required by the min & max. + requiredDepth := bitDepthInt64(min - bsig.Base) + if v := bitDepthInt64(max - bsig.Base); v > requiredDepth { + requiredDepth = v + } + + // Increase bit depth if required. + if requiredDepth > bsig.BitDepth { + if err := func() error { + f.mu.Lock() + defer f.mu.Unlock() + bsig.BitDepth = requiredDepth + f.options.BitDepth = requiredDepth + return f.saveMeta() + }(); err != nil { + return errors.Wrap(err, "increasing bsi bit depth") + } + } + // Split import data by fragment. dataByFragment := make(map[importKey]importValueData) for i := range columnIDs { @@ -1169,7 +1282,6 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO // Import into each fragment. for key, data := range dataByFragment { - // The view must already exist (i.e. we can't create it) // because we need to know bitDepth (based on min/max value). view, err := f.createViewIfNotExists(key.View) @@ -1182,12 +1294,12 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return errors.Wrap(err, "creating fragment") } - baseValues := make([]uint64, len(data.Values)) + baseValues := make([]int64, len(data.Values)) for i, value := range data.Values { - baseValues[i] = uint64(value - bsig.Min) + baseValues[i] = value - bsig.Base } - if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth(), options.Clear); err != nil { + if err := frag.importValue(data.ColumnIDs, baseValues, requiredDepth, options.Clear); err != nil { return err } } @@ -1195,10 +1307,14 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return nil } -func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear bool) error { +func (f *Field) importRoaring(ctx context.Context, data []byte, shard uint64, viewName string, clear bool) error { + span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaring") + defer span.Finish() + if viewName == "" { viewName = viewStandard } + span.LogKV("view", viewName, "bytes", len(data), "shard", shard) view, err := f.createViewIfNotExists(viewName) if err != nil { return errors.Wrap(err, "creating view") @@ -1209,7 +1325,7 @@ func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaring(data, clear); err != nil { + if err := frag.importRoaring(ctx, data, clear); err != nil { return err } @@ -1237,6 +1353,8 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FieldOptions represents options to set when initializing a field. type FieldOptions struct { + Base int64 `json:"base,omitempty"` + BitDepth uint `json:"bitDepth,omitempty"` Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` Keys bool `json:"keys"` @@ -1273,6 +1391,8 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Type: o.Type, CacheType: o.CacheType, CacheSize: o.CacheSize, + Base: o.Base, + BitDepth: uint64(o.BitDepth), Min: o.Min, Max: o.Max, TimeQuantum: string(o.TimeQuantum), @@ -1281,6 +1401,9 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { } } +// MarshalJSON marshals FieldOptions to JSON such that +// only those attributes associated to the field type +// are included. func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { case FieldTypeSet: @@ -1297,12 +1420,16 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }) case FieldTypeInt: return json.Marshal(struct { - Type string `json:"type"` - Min int64 `json:"min"` - Max int64 `json:"max"` - Keys bool `json:"keys"` + Type string `json:"type"` + Base int64 `json:"base"` + BitDepth uint `json:"bitDepth"` + Min int64 `json:"min"` + Max int64 `json:"max"` + Keys bool `json:"keys"` }{ o.Type, + o.Base, + o.BitDepth, o.Min, o.Max, o.Keys, @@ -1357,20 +1484,12 @@ func isValidBSIGroupType(v string) bool { // bsiGroup represents a group of range-encoded rows on a field. type bsiGroup struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -// BitDepth returns the number of bits required to store a value between min & max. -func (b *bsiGroup) BitDepth() uint { - for i := uint(0); i < 63; i++ { - if b.Max-b.Min < (1 << i) { - return i - } - } - return 63 + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + Base int64 `json:"base,omitempty"` + BitDepth uint `json:"bitDepth,omitempty"` } // baseValue adjusts the value to align with the range for Field for a certain @@ -1385,46 +1504,47 @@ func (b *bsiGroup) BitDepth() uint { // In order to make this work, we effectively need to change the operator to LTE. // Executor.executeBSIGroupRangeShard() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. -func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { +func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue int64, outOfRange bool) { + min, max := b.bitDepthMin(), b.bitDepthMax() + if op == pql.GT || op == pql.GTE { - if value > b.Max { + if value > max { return baseValue, true - } else if value > b.Min { - baseValue = uint64(value - b.Min) + } else if value > min { + baseValue = int64(value - b.Base) } } else if op == pql.LT || op == pql.LTE { - if value < b.Min { + if value < min { return baseValue, true - } else if value > b.Max { - baseValue = uint64(b.Max - b.Min) + } else if value > max { + baseValue = int64(max - b.Base) } else { - baseValue = uint64(value - b.Min) + baseValue = int64(value - b.Base) } } else if op == pql.EQ || op == pql.NEQ { - if value < b.Min || value > b.Max { + if value < min || value > max { return baseValue, true } - baseValue = uint64(value - b.Min) + baseValue = int64(value - b.Base) } return baseValue, false } // baseValueBetween adjusts the min/max value to align with the range for Field. -func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { - if max < b.Min || min > b.Max { - return baseValueMin, baseValueMax, true +func (b *bsiGroup) baseValueBetween(lo, hi int64) (baseValueLo, baseValueHi int64, outOfRange bool) { + min, max := b.bitDepthMin(), b.bitDepthMax() + if hi < min || lo > max { + return 0, 0, true } - // Adjust min/max to range. - if min > b.Min { - baseValueMin = uint64(min - b.Min) + + // Limit lo/hi to possible bit range. + if lo < min { + lo = min } - // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > b.Max { - baseValueMax = uint64(b.Max - b.Min) - } else if max > b.Min { - baseValueMax = uint64(max - b.Min) + if hi > max { + hi = max } - return baseValueMin, baseValueMax, false + return lo - b.Base, hi - b.Base, false } func (b *bsiGroup) validate() error { @@ -1432,12 +1552,20 @@ func (b *bsiGroup) validate() error { return ErrBSIGroupNameRequired } else if !isValidBSIGroupType(b.Type) { return ErrInvalidBSIGroupType - } else if b.Min > b.Max { - return ErrInvalidBSIGroupRange } return nil } +// bitDepthMin returns the minimum value possible for the current bit depth. +func (b *bsiGroup) bitDepthMin() int64 { + return b.Base - (1 << b.BitDepth) + 1 +} + +// bitDepthMax returns the maximum value possible for the current bit depth. +func (b *bsiGroup) bitDepthMax() int64 { + return b.Base + (1 << b.BitDepth) - 1 +} + // Cache types. const ( CacheTypeLRU = "lru" @@ -1454,3 +1582,21 @@ func isValidCacheType(v string) bool { return false } } + +// bitDepth returns the number of bits required to store a value. +func bitDepth(v uint64) uint { + for i := uint(0); i < 63; i++ { + if v < (1 << i) { + return i + } + } + return 63 +} + +// bitDepthInt64 returns the required bit depth for abs(v). +func bitDepthInt64(v int64) uint { + if v < 0 { + return bitDepth(uint64(-v)) + } + return bitDepth(uint64(v)) +} diff --git a/field_internal_test.go b/field_internal_test.go index 0a26225bb..13851a493 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -15,7 +15,9 @@ package pilosa import ( + "fmt" "io/ioutil" + "math" "os" "reflect" "testing" @@ -28,125 +30,127 @@ import ( // Ensure a bsiGroup can adjust to its baseValue. func TestBSIGroup_BaseValue(t *testing.T) { b0 := &bsiGroup{ - Name: "b0", - Type: bsiGroupTypeInt, - Min: -100, - Max: 900, + Name: "b0", + Type: bsiGroupTypeInt, + Base: -100, + BitDepth: 10, + Min: -1000, + Max: 1000, } b1 := &bsiGroup{ - Name: "b1", - Type: bsiGroupTypeInt, - Min: 0, - Max: 1000, + Name: "b1", + Type: bsiGroupTypeInt, + Base: 0, + BitDepth: 8, + Min: -255, + Max: 255, } - b2 := &bsiGroup{ - Name: "b2", - Type: bsiGroupTypeInt, - Min: 100, - Max: 1100, + Name: "b2", + Type: bsiGroupTypeInt, + Base: 100, + BitDepth: 11, + Min: math.MinInt64, + Max: math.MaxInt64, } t.Run("Normal Condition", func(t *testing.T) { - - for _, tt := range []struct { + for i, tt := range []struct { f *bsiGroup op pql.Token val int64 - expBaseValue uint64 + expBaseValue int64 expOutOfRange bool }{ // LT {b0, pql.LT, 5, 105, false}, {b0, pql.LT, -8, 92, false}, - {b0, pql.LT, -108, 0, true}, - {b0, pql.LT, 1005, 1000, false}, + {b0, pql.LT, -108, -8, false}, + {b0, pql.LT, 1005, 1023, false}, {b0, pql.LT, 0, 100, false}, {b1, pql.LT, 5, 5, false}, - {b1, pql.LT, -8, 0, true}, - {b1, pql.LT, 1005, 1000, false}, + {b1, pql.LT, -8, -8, false}, + {b1, pql.LT, 1005, 255, false}, {b1, pql.LT, 0, 0, false}, - {b2, pql.LT, 5, 0, true}, - {b2, pql.LT, -8, 0, true}, + {b2, pql.LT, 5, -95, false}, + {b2, pql.LT, -8, -108, false}, {b2, pql.LT, 105, 5, false}, - {b2, pql.LT, 1105, 1000, false}, + {b2, pql.LT, 1105, 1005, false}, // GT - {b0, pql.GT, -105, 0, false}, + {b0, pql.GT, -5, 95, false}, {b0, pql.GT, 5, 105, false}, - {b0, pql.GT, 905, 0, true}, + {b0, pql.GT, 905, 1005, false}, {b0, pql.GT, 0, 100, false}, {b1, pql.GT, 5, 5, false}, - {b1, pql.GT, -8, 0, false}, + {b1, pql.GT, -8, -8, false}, {b1, pql.GT, 1005, 0, true}, {b1, pql.GT, 0, 0, false}, - {b2, pql.GT, 5, 0, false}, - {b2, pql.GT, -8, 0, false}, + {b2, pql.GT, 5, -95, false}, + {b2, pql.GT, -8, -108, false}, {b2, pql.GT, 105, 5, false}, - {b2, pql.GT, 1105, 0, true}, + {b2, pql.GT, 1105, 1005, false}, // EQ - {b0, pql.EQ, -105, 0, true}, + {b0, pql.EQ, -105, -5, false}, {b0, pql.EQ, 5, 105, false}, - {b0, pql.EQ, 905, 0, true}, + {b0, pql.EQ, 905, 1005, false}, {b0, pql.EQ, 0, 100, false}, {b1, pql.EQ, 5, 5, false}, - {b1, pql.EQ, -8, 0, true}, + {b1, pql.EQ, -8, -8, false}, {b1, pql.EQ, 1005, 0, true}, {b1, pql.EQ, 0, 0, false}, - {b2, pql.EQ, 5, 0, true}, - {b2, pql.EQ, -8, 0, true}, + {b2, pql.EQ, 5, -95, false}, + {b2, pql.EQ, -8, -108, false}, {b2, pql.EQ, 105, 5, false}, - {b2, pql.EQ, 1105, 0, true}, + {b2, pql.EQ, 1105, 1005, false}, } { - bv, oor := tt.f.baseValue(tt.op, tt.val) - if oor != tt.expOutOfRange { - t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) - } + t.Run(fmt.Sprint(i), func(t *testing.T) { + bv, oor := tt.f.baseValue(tt.op, tt.val) + if oor != tt.expOutOfRange || !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Errorf("%s) baseValue(%s, %v)=(%v, %v), expected (%v, %v)", tt.f.Name, tt.op, tt.val, bv, oor, tt.expBaseValue, tt.expOutOfRange) + } + }) } }) - t.Run("Betwween Condition", func(t *testing.T) { - for _, tt := range []struct { + t.Run("Between Condition", func(t *testing.T) { + for i, tt := range []struct { f *bsiGroup predMin int64 predMax int64 - expBaseValueMin uint64 - expBaseValueMax uint64 + expBaseValueMin int64 + expBaseValueMax int64 expOutOfRange bool }{ - {b0, -205, -105, 0, 0, true}, - {b0, -105, 80, 0, 180, false}, + {b0, -205, -105, -105, -5, false}, + {b0, -105, 80, -5, 180, false}, {b0, 5, 20, 105, 120, false}, - {b0, 20, 1005, 120, 1000, false}, + {b0, 20, 1005, 120, 1023, false}, {b0, 1005, 2000, 0, 0, true}, - {b1, -105, -5, 0, 0, true}, - {b1, -5, 20, 0, 20, false}, + {b1, -105, -5, -105, -5, false}, + {b1, -5, 20, -5, 20, false}, {b1, 5, 20, 5, 20, false}, - {b1, 20, 1005, 20, 1000, false}, + {b1, 20, 1005, 20, 255, false}, {b1, 1005, 2000, 0, 0, true}, - {b2, 5, 95, 0, 0, true}, - {b2, 95, 120, 0, 20, false}, + {b2, 5, 95, -95, -5, false}, + {b2, 95, 120, -5, 20, false}, {b2, 105, 120, 5, 20, false}, - {b2, 120, 1105, 20, 1000, false}, - {b2, 1105, 2000, 0, 0, true}, + {b2, 120, 1105, 20, 1005, false}, + {b2, 1105, 2000, 1005, 1900, false}, } { min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax) - if oor != tt.expOutOfRange { - t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { - t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) + if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) || oor != tt.expOutOfRange { + t.Errorf("%d. %s) baseValueBetween(%v, %v)=(%v, %v, %v), expected (%v, %v, %v)", i, tt.f.Name, tt.predMin, tt.predMax, min, max, oor, tt.expBaseValueMin, tt.expBaseValueMax, tt.expOutOfRange) } } }) @@ -370,7 +374,10 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) { // bm represents remote available shards. bm := roaring.NewBitmap() for i := uint64(0); i < 1204; i += 2 { - bm.Add(i) + _, err := bm.Add(i) + if err != nil { + t.Fatalf("adding bits: %v", err) + } } if err := f.AddRemoteAvailableShards(bm); err != nil { @@ -386,7 +393,10 @@ func TestField_PersistAvailableShardsFootprint(t *testing.T) { bm1 := roaring.NewBitmap() for i := uint64(1); i < 1204; i += 2 { - bm1.Add(i) + _, err := bm1.Add(i) + if err != nil { + t.Fatalf("adding bits: %v", err) + } } if err := f.AddRemoteAvailableShards(bm1); err != nil { diff --git a/field_test.go b/field_test.go index 2a0efc0b9..c60c9a833 100644 --- a/field_test.go +++ b/field_test.go @@ -16,6 +16,7 @@ package pilosa_test import ( "io/ioutil" + "math" "testing" "github.com/google/go-cmp/cmp" @@ -30,7 +31,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -63,7 +64,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatal(err) } @@ -208,17 +209,20 @@ func TestField_AvailableShards(t *testing.T) { } // Set remote shards and verify. - f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)) + if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil { + t.Fatalf("adding remote shards: %v", err) + } if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" { t.Fatal(diff) } // Delete shards; only local shards should remain. - f.RemoveAvailableShard(0) - f.RemoveAvailableShard(1) - f.RemoveAvailableShard(2) - f.RemoveAvailableShard(3) - f.RemoveAvailableShard(4) + for i := uint64(0); i < 5; i++ { + err := f.RemoveAvailableShard(i) + if err != nil { + t.Fatalf("removing shard %d: %v", i, err) + } + } if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) } diff --git a/fragment.go b/fragment.go index 4769ce700..eaccf7b86 100644 --- a/fragment.go +++ b/fragment.go @@ -40,6 +40,7 @@ import ( "github.com/pilosa/pilosa/logger" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" + "github.com/pilosa/pilosa/shardwidth" "github.com/pilosa/pilosa/stats" "github.com/pilosa/pilosa/syswrap" "github.com/pilosa/pilosa/tracing" @@ -48,8 +49,8 @@ import ( const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. - shardWidthExponent = 20 - ShardWidth = 1 << shardWidthExponent + // shardWidthExponent = 20 // set in shardwidthNN.go files + ShardWidth = 1 << shardwidth.Exponent // shardVsContainerExponent is the power of 2 of ShardWith minus the power // of two of roaring container width (which is 16). @@ -59,7 +60,7 @@ const ( // which a given container is in means dividing by the number of rows per // container which is performantly expressed as a right shift by this // exponent. - shardVsContainerExponent = shardWidthExponent - 16 + shardVsContainerExponent = shardwidth.Exponent - 16 // width of roaring containers is 2^16 containerWidth = 1 << 16 @@ -82,6 +83,14 @@ const ( // Row ids used for boolean fields. falseRowID = uint64(0) trueRowID = uint64(1) + + // BSI bits used to check existence & sign. + bsiExistsBit = 0 + bsiSignBit = 1 + bsiOffsetBit = 2 + + // Roaring bitmap flags. + roaringFlagBSIv2 = 0x01 // indicates version using low bit for existence ) // fragment represents the intersection of a field and shard in an index. @@ -96,6 +105,7 @@ type fragment struct { // File-backed storage path string + flags byte // user-defined flags passed to roaring file *os.File storage *roaring.Bitmap storageData []byte @@ -135,13 +145,14 @@ type fragment struct { } // newFragment returns a new instance of Fragment. -func newFragment(path, index, field, view string, shard uint64) *fragment { +func newFragment(path, index, field, view string, shard uint64, flags byte) *fragment { return &fragment{ path: path, index: index, field: field, view: view, shard: shard, + flags: flags, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -162,11 +173,13 @@ func (f *fragment) Open() error { if err := func() error { // Initialize storage in a function so we can close if anything goes wrong. + f.Logger.Debugf("open storage for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) if err := f.openStorage(); err != nil { return errors.Wrap(err, "opening storage") } // Fill cache with rows persisted to disk. + f.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) if err := f.openCache(); err != nil { return errors.Wrap(err, "opening cache") } @@ -184,6 +197,7 @@ func (f *fragment) Open() error { return err } + f.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) return nil } @@ -204,6 +218,7 @@ func (f *fragment) openStorage() error { // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() + f.storage.Flags = f.flags } // Open the data file to be mmap'd and used as an ops log. file, mustClose, err := syswrap.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) @@ -407,19 +422,18 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row { func (f *fragment) rowFromStorage(rowID uint64) *Row { // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by container width. + // + // Note that OffsetRange now returns a new bitmap which uses frozen + // containers which will use copy-on-write semantics. The actual bitmap + // and Containers object are new and not shared, but the containers are + // shared. data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) - // Reference bitmap subrange in storage. We Clone() data because otherwise - // row will contain pointers to containers in storage. This causes - // unexpected results when we cache the row and try to use it later. - // Basically, since we return the Row and release the fragment lock, the - // underlying fragment storage could be changed or snapshotted and thrown - // out at any point. row := &Row{ segments: []rowSegment{{ - data: *data.Clone(), + data: data, shard: f.shard, - writable: false, // this Row will probably be cached and shared, so it must be read only. + writable: true, }}, } row.invalidateCount() @@ -490,12 +504,15 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err return false, errors.Wrap(err, "incrementing") } - // Get the row from row cache or fragment.storage. - row := f.unprotectedRow(rowID) - row.SetBit(columnID) - - // Update the cache. - f.cache.Add(rowID, row.Count()) + // If we're using a cache, update it. Otherwise skip the + // possibly-expensive count operation. + if f.CacheType != CacheTypeNone { + n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.Add(rowID, n) + } + // Drop the rowCache entry; it's wrong, and we don't want to force + // a new copy if no one's reading it. + f.rowCache.Add(rowID, nil) f.stats.Count("setBit", 1, 0.001) @@ -551,12 +568,15 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return false, errors.Wrap(err, "incrementing") } - // Get the row from cache or fragment.storage. - row := f.unprotectedRow(rowID) - row.clearBit(columnID) - - // Update the cache. - f.cache.Add(rowID, row.Count()) + // If we're using a cache, update it. Otherwise skip the + // possibly-expensive count operation. + if f.CacheType != CacheTypeNone { + n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.Add(rowID, n) + } + // Drop the rowCache entry; it's wrong, and we don't want to force + // a new copy if no one's reading it. + f.rowCache.Add(rowID, nil) f.stats.Count("clearBit", 1, 1.0) @@ -675,12 +695,12 @@ func (f *fragment) bit(rowID, columnID uint64) (bool, error) { } // value uses a column of bits to read a multi-bit value. -func (f *fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +func (f *fragment) value(columnID uint64, bitDepth uint) (value int64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() // If existence bit is unset then ignore remaining bits. - if v, err := f.bit(uint64(bitDepth), columnID); err != nil { + if v, err := f.bit(bsiExistsBit, columnID); err != nil { return 0, false, errors.Wrap(err, "getting existence bit") } else if !v { return 0, false, nil @@ -688,55 +708,75 @@ func (f *fragment) value(columnID uint64, bitDepth uint) (value uint64, exists b // Compute other bits into a value. for i := uint(0); i < bitDepth; i++ { - if v, err := f.bit(uint64(i), columnID); err != nil { + if v, err := f.bit(uint64(bsiOffsetBit+i), columnID); err != nil { return 0, false, errors.Wrapf(err, "getting value bit %d", i) } else if v { value |= (1 << i) } } + // Negate if sign bit set. + if v, err := f.bit(bsiSignBit, columnID); err != nil { + return 0, false, errors.Wrap(err, "getting sign bit") + } else if v { + value = -value + } + return value, true, nil } // clearValue uses a column of bits to clear a multi-bit value. -func (f *fragment) clearValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (f *fragment) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { return f.setValueBase(columnID, bitDepth, value, true) } // setValue uses a column of bits to set a multi-bit value. -func (f *fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (f *fragment) setValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { return f.setValueBase(columnID, bitDepth, value, false) } -func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value uint64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) { +func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) { + // Convert value to an unsigned representation. + uvalue := uint64(value) + if value < 0 { + uvalue = uint64(-value) + } + + // Mark value as set. + if bit, err := f.pos(bsiExistsBit, columnID); err != nil { + return toSet, toClear, errors.Wrap(err, "getting not-null pos") + } else if clear { + toClear = append(toClear, bit) + } else { + toSet = append(toSet, bit) + } + + // Mark sign. + if bit, err := f.pos(bsiSignBit, columnID); err != nil { + return toSet, toClear, errors.Wrap(err, "getting sign pos") + } else if value >= 0 || clear { + toClear = append(toClear, bit) + } else { + toSet = append(toSet, bit) + } + for i := uint(0); i < bitDepth; i++ { - bit, err := f.pos(uint64(i), columnID) + bit, err := f.pos(uint64(bsiOffsetBit+i), columnID) if err != nil { return toSet, toClear, errors.Wrap(err, "getting pos") } - if value&(1<= 0 || clear { + if c, err := f.unprotectedClearBit(uint64(bsiSignBit), columnID); err != nil { + return changed, errors.Wrap(err, "clearing sign") + } else if c { + changed = true + } + } else { + if c, err := f.unprotectedSetBit(uint64(bsiSignBit), columnID); err != nil { + return changed, errors.Wrap(err, "marking sign") + } else if c { + changed = true + } + } + return changed, nil } // importSetValue is a more efficient SetValue just for imports. -func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value uint64, clear bool) (changed bool, err error) { // nolint: unparam +func (f *fragment) importSetValue(columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) { // nolint: unparam + // Convert value to an unsigned representation. + uvalue := uint64(value) + if value < 0 { + uvalue = uint64(-value) + } + for i := uint(0); i < bitDepth; i++ { - if value&(1<= 0 || clear { + if c, err := f.storage.Remove(p); err != nil { + return changed, errors.Wrap(err, "removing sign from storage") + } else if c { + changed = true + } + } else { + if c, err := f.storage.Add(p); err != nil { + return changed, errors.Wrap(err, "adding sign to storage") + } else if c { + changed = true + } + } + return changed, nil } // sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (f *fragment) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { // Compute count based on the existence row. - consider := f.row(uint64(bitDepth)) + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } count = consider.Count() + // Determine positive & negative sets. + nrow := f.row(bsiSignBit) + prow := consider.Difference(nrow) + // Compute the sum based on the bit count of each row multiplied by the // place value of each row. For example, 10 bits in the 1's place plus // 4 bits in the 2's place plus 3 bits in the 4's place equals a total @@ -846,11 +929,16 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 // - var cnt uint64 + // Execute once for positive numbers and once for negative. Subtract the + // negative sum from the positive sum. for i := uint(0); i < bitDepth; i++ { - row := f.row(uint64(i)) - cnt = row.intersectionCount(consider) - sum += (1 << i) * cnt + row := f.row(uint64(bsiOffsetBit + i)) + + psum := int64((1 << i) * row.intersectionCount(prow)) + nsum := int64((1 << i) * row.intersectionCount(nrow)) + + // Squash to reduce the possibility of overflow. + sum += psum - nsum } return sum, count, nil @@ -858,9 +946,8 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error // min returns the min of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) { - - consider := f.row(uint64(bitDepth)) +func (f *fragment) min(filter *Row, bitDepth uint) (min int64, count uint64, err error) { + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } @@ -870,58 +957,79 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error return 0, 0, nil } - for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.row(uint64(ii)) + // If we have negative values, we should find the highest unsigned value + // from that set, then negate it, and return it. For example, if values + // (-1, -2) exist, they are stored unsigned (1,2) with a negative sign bit + // set. We take the highest of that set (2) and negate it and return it. + if row := f.row(bsiSignBit).Intersect(consider); row.Any() { + min, count := f.maxUnsigned(row, bitDepth) + return -min, count, nil + } - x := consider.Difference(row) - count = x.Count() + // Otherwise find lowest positive number. + min, count = f.minUnsigned(consider, bitDepth) + return min, count, nil +} + +// minUnsigned the lowest value without considering the sign bit. Filter is required. +func (f *fragment) minUnsigned(filter *Row, bitDepth uint) (min int64, count uint64) { + for i := int(bitDepth - 1); i >= 0; i-- { + row := filter.Difference(f.row(uint64(bsiOffsetBit + i))) + count = row.Count() if count > 0 { - consider = x + filter = row } else { - min += (1 << ii) - if ii == 0 { - count = consider.Count() + min += (1 << uint(i)) + if i == 0 { + count = filter.Count() } } } - - return min, count, nil + return min, count } // max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - - consider := f.row(uint64(bitDepth)) +func (f *fragment) max(filter *Row, bitDepth uint) (max int64, count uint64, err error) { + consider := f.row(bsiExistsBit) if filter != nil { consider = consider.Intersect(filter) } // If there are no columns to consider, return early. - if consider.Count() == 0 { + if !consider.Any() { return 0, 0, nil } - for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.row(uint64(ii)) - - x := row.Intersect(consider) - count = x.Count() - if count > 0 { - max += (1 << ii) - consider = x - } else if ii == 0 { - count = consider.Count() - } + // Find lowest negative number w/o sign and negate, if no positives are available. + pos := consider.Difference(f.row(bsiSignBit)) + if !pos.Any() { + max, count = f.minUnsigned(consider, bitDepth) + return -max, count, nil } + // Otherwise find highest positive number. + max, count = f.maxUnsigned(pos, bitDepth) return max, count, nil } +// maxUnsigned the highest value without considering the sign bit. Filter is required. +func (f *fragment) maxUnsigned(filter *Row, bitDepth uint) (max int64, count uint64) { + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.row(uint64(bsiOffsetBit + i)).Intersect(filter) + count = row.Count() + if count > 0 { + max += (1 << uint(i)) + filter = row + } else if i == 0 { + count = filter.Count() + } + } + return max, count +} + // rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. -func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { switch op { case pql.EQ: return f.rangeEQ(bitDepth, predicate) @@ -936,14 +1044,23 @@ func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, } } -func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) + + // Filter to only positive/negative numbers. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + b = b.Intersect(f.row(bsiSignBit)) // only negatives + } else { + b = b.Difference(f.row(bsiSignBit)) // only positives + } // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) - bit := (predicate >> uint(i)) & 1 + row := f.row(uint64(bsiOffsetBit + i)) + bit := (upredicate >> uint(i)) & 1 if bit == 1 { b = b.Intersect(row) @@ -955,9 +1072,9 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) // Get the equal bitmap. eq, err := f.rangeEQ(bitDepth, predicate) @@ -971,22 +1088,44 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - +func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { // Start with set of columns with values set. - b := f.row(uint64(bitDepth)) + b := f.row(bsiExistsBit) + + // Create predicate without sign bit. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + } + + // If predicate is positive, return all positives less than predicate and all negatives. + if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + neg := f.row(bsiSignBit) + return neg.Union(pos), nil + } + + // Otherwise if predicate is negative, return all negatives greater than upredicate. + return f.rangeGTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) +} + +// rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. +func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + keep := NewRow() // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. if leadingZeros { if bit == 0 { - b = b.Difference(row) + filter = filter.Difference(row) continue } else { leadingZeros = false @@ -1000,32 +1139,54 @@ func (f *fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) if bit == 0 { return keep, nil } - return b.Difference(row.Difference(keep)), nil + return filter.Difference(row.Difference(keep)), nil } // If bit is zero then remove all set columns not in excluded bitmap. if bit == 0 { - b = b.Difference(row.Difference(keep)) + filter = filter.Difference(row.Difference(keep)) continue } // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Difference(row)) + keep = keep.Union(filter.Difference(row)) } } - return b, nil + return filter, nil } -func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.row(uint64(bitDepth)) +func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + b := f.row(bsiExistsBit) + + // Create predicate without sign bit. + upredicate := uint64(predicate) + if predicate < 0 { + upredicate = uint64(-predicate) + } + + // If predicate is positive, return all positives greater than predicate. + if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { + return f.rangeGTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + } + + // If predicate is negative, return all negatives less than than upredicate and all positives. + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + pos := b.Difference(f.row(bsiSignBit)) + return pos.Union(neg), nil +} + +func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -1035,68 +1196,102 @@ func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) if bit == 1 { return keep, nil } - return b.Difference(b.Difference(row).Difference(keep)), nil + return filter.Difference(filter.Difference(row).Difference(keep)), nil } // If bit is set then remove all unset columns not already kept. if bit == 1 { - b = b.Difference(b.Difference(row).Difference(keep)) + filter = filter.Difference(filter.Difference(row).Difference(keep)) continue } // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Intersect(row)) + keep = keep.Union(filter.Intersect(row)) } } - return b, nil + return filter, nil } -// notNull returns the not-null row (stored at bitDepth). -func (f *fragment) notNull(bitDepth uint) (*Row, error) { - return f.row(uint64(bitDepth)), nil +// notNull returns the exists row. +func (f *fragment) notNull() (*Row, error) { + return f.row(bsiExistsBit), nil } // rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. -func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.row(uint64(bitDepth)) +func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) (*Row, error) { + b := f.row(bsiExistsBit) + + // Convert predicates to unsigned values. + upredicateMin, upredicateMax := uint64(predicateMin), uint64(predicateMax) + if predicateMin < 0 { + upredicateMin = uint64(-predicateMin) + } + if predicateMax < 0 { + upredicateMax = uint64(-predicateMax) + } + + // Handle positive-only values. + if predicateMin >= 0 { + return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) + } + + // Handle negative-only values. Swap unsigned min/max predicates. + if predicateMax < 0 { + return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin) + } + + // If predicate crosses positive/negative boundary then handle separately and union. + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) + if err != nil { + return nil, err + } + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) + if err != nil { + return nil, err + } + return pos.Union(neg), nil +} + +// rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. +func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.row(uint64(i)) + row := f.row(uint64(bsiOffsetBit + i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 // GTE predicateMin // If bit is set then remove all unset columns not already kept. if bit1 == 1 { - b = b.Difference(b.Difference(row).Difference(keep1)) + filter = filter.Difference(filter.Difference(row).Difference(keep1)) } else { // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep1 = keep1.Union(b.Intersect(row)) + keep1 = keep1.Union(filter.Intersect(row)) } } - // LTE predicateMin + // LTE predicateMax // If bit is zero then remove all set bits not in excluded bitmap. if bit2 == 0 { - b = b.Difference(row.Difference(keep2)) + filter = filter.Difference(row.Difference(keep2)) } else { // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { - keep2 = keep2.Union(b.Difference(row)) + keep2 = keep2.Union(filter.Difference(row)) } } } - return b, nil + return filter, nil } // pos translates the row ID and column ID into a position in the storage bitmap. @@ -1326,7 +1521,7 @@ type topOptions struct { func (f *fragment) Checksum() []byte { h := xxhash.New() for _, block := range f.Blocks() { - h.Write(block.Checksum) + _, _ = h.Write(block.Checksum) } return h.Sum(nil) } @@ -1659,9 +1854,7 @@ func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct f.cache.BulkAdd(rowID, n) if smallWrite { - if _, ok := f.rowCache.Fetch(rowID); ok { // we won't update the rowCache if it wasn't already in there. - f.rowCache.Add(rowID, f.rowFromStorage(rowID)) - } + f.rowCache.Add(rowID, nil) } } @@ -1731,7 +1924,7 @@ func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { return errors.Wrap(f.importPositions(toSet, toClear, rowSet), "importing positions") } -func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth uint, clear bool) error { +func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { // TODO figure out how to avoid re-allocating these each time. Probably // possible to store them on the fragment with a capacity based on // MaxOpN. For now, we know that the total number of bits to be @@ -1747,6 +1940,7 @@ func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth ui if _, ok := colSet[columnID]; ok { continue } + colSet[columnID] = struct{}{} toSet, toClear, err = f.positionsForValue(columnID, bitDepth, value, clear, toSet, toClear) if err != nil { @@ -1768,7 +1962,7 @@ func (f *fragment) importValueSmallWrite(columnIDs, values []uint64, bitDepth ui } // importValue bulk imports a set of range-encoded values. -func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear bool) error { +func (f *fragment) importValue(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error { f.mu.Lock() defer f.mu.Unlock() @@ -1779,7 +1973,6 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear if len(columnIDs)*int(bitDepth+1)+f.opN < f.MaxOpN { return errors.Wrap(f.importValueSmallWrite(columnIDs, values, bitDepth, clear), "import small write") - } f.storage.OpWriter = nil @@ -1807,11 +2000,17 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint, clear // importRoaring imports from the official roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. -func (f *fragment) importRoaring(data []byte, clear bool) error { +func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) error { + span, ctx := tracing.StartSpanFromContext(ctx, "fragment.importRoaring") + defer span.Finish() + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.AcquireFragmentLock") f.mu.Lock() defer f.mu.Unlock() + span.Finish() bm := roaring.NewBTreeBitmap() + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.UnmarshalBinary") err := bm.UnmarshalBinary(data) + span.Finish() if err != nil { return err } @@ -1847,16 +2046,25 @@ func (f *fragment) importRoaring(data []byte, clear bool) error { if clear { toSet, toClear = toClear, toSet } - return f.importPositions(toSet, toClear, rowSet) + span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.ImportPositions") + err := f.importPositions(toSet, toClear, rowSet) + span.Finish() + return err } if clear { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringDifference") bm = f.storage.Difference(bm) + span.Finish() } else if f.storage.Containers.Size() >= bm.Containers.Size() { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringStorageUIP") f.storage.UnionInPlace(bm) bm = f.storage + span.Finish() } else { + span, ctx = tracing.StartSpanFromContext(ctx, "importRoaringBitmapUIP") bm.UnionInPlace(f.storage) + span.Finish() } for rowID := range rowSet { @@ -1865,7 +2073,10 @@ func (f *fragment) importRoaring(data []byte, clear bool) error { } f.cache.Recalculate() - err = unprotectedWriteToFragment(f, bm) + span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.WriteToFragment") + n, err := unprotectedWriteToFragment(f, bm) + span.LogKV("bytesWritten", n) + span.Finish() return err } @@ -1896,12 +2107,13 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg } func (f *fragment) snapshot() error { - return unprotectedWriteToFragment(f, f.storage) + _, err := unprotectedWriteToFragment(f, f.storage) + return err } // unprotectedWriteToFragment writes the fragment f with bm as the data. It is unprotected, and // f.mu must be locked when calling it. -func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer +func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) (n int64, err error) { // nolint: interfacer completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() @@ -1911,39 +2123,39 @@ func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // noli snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { - return fmt.Errorf("create snapshot file: %s", err) + return n, fmt.Errorf("create snapshot file: %s", err) } defer file.Close() // Write storage to snapshot. bw := bufio.NewWriter(file) - if _, err := bm.WriteTo(bw); err != nil { - return fmt.Errorf("snapshot write to: %s", err) + if n, err = bm.WriteTo(bw); err != nil { + return n, fmt.Errorf("snapshot write to: %s", err) } if err := bw.Flush(); err != nil { - return fmt.Errorf("flush: %s", err) + return n, fmt.Errorf("flush: %s", err) } // Close current storage. if err := f.closeStorage(); err != nil { - return fmt.Errorf("close storage: %s", err) + return n, fmt.Errorf("close storage: %s", err) } // Move snapshot to data file location. if err := os.Rename(snapshotPath, f.path); err != nil { - return fmt.Errorf("rename snapshot: %s", err) + return n, fmt.Errorf("rename snapshot: %s", err) } // Reopen storage. if err := f.openStorage(); err != nil { - return fmt.Errorf("open storage: %s", err) + return n, fmt.Errorf("open storage: %s", err) } // Reset operation count. f.opN = 0 - return nil + return n, nil } // RecalculateCache rebuilds the cache regardless of invalidate time delay. @@ -2265,6 +2477,51 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { return rows } +// upgradeRoaringBSIv2 upgrades a fragment that contains old BSI formatting +// to a new BSI format (v2). The new format moves the "exists" bit to the +// beginning & adds a negative sign bit. +func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) { + // If flag set, already upgraded. Exit. + if f.storage.Flags&roaringFlagBSIv2 == 1 { + return "", nil + } + + other := roaring.NewBitmap() + other.Flags = roaringFlagBSIv2 + func() { + f.mu.Lock() + defer f.mu.Unlock() + + f.storage.ForEach(func(i uint64) { + rowID, columnID := i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth) + if rowID == uint64(bitDepth) { + _, _ = other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning + } else { + _, _ = other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up + } + }) + }() + + // Create temporary file next to existing file. + newPath := f.path + ".tmp" + file, err := os.OpenFile(newPath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return "", err + } + defer file.Close() + + // Write & flush to temporary file. + if _, err := other.WriteTo(file); err != nil { + return "", err + } else if err := file.Sync(); err != nil { + return "", err + } else if err := file.Close(); err != nil { + return "", err + } + + return newPath, nil +} + type rowIterator struct { f *fragment rowIDs []uint64 @@ -2297,7 +2554,7 @@ func (ri *rowIterator) Next() (r *Row, rowID uint64, wrapped bool) { } rowID = ri.rowIDs[ri.cur] r = ri.f.row(rowID) - ri.cur += 1 + ri.cur++ return r, rowID, wrapped } @@ -2330,7 +2587,7 @@ func (h *blockHasher) Sum() []byte { func (h *blockHasher) WriteValue(v uint64) { binary.BigEndian.PutUint64(h.buf[:], v) - h.hash.Write(h.buf[:]) + _, _ = h.hash.Write(h.buf[:]) } // fragmentSyncer syncs a local fragment to one on a remote host. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 5186d6da1..e45d3cce8 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -16,6 +16,7 @@ package pilosa import ( "bytes" + "context" "flag" "fmt" "io" @@ -25,6 +26,7 @@ import ( "os" "reflect" "sort" + "sync/atomic" "testing" "testing/quick" @@ -33,6 +35,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // Test flags @@ -102,6 +105,44 @@ func TestFragment_ClearBit(t *testing.T) { } } +// What about rowcache timing. +func TestFragment_RowcacheMap(t *testing.T) { + var done int64 + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + ch := make(chan struct{}) + + for i := 0; i < f.MaxOpN; i++ { + _, _ = f.setBit(0, uint64(i*32)) + } + // force snapshot so we get a mmapped row... + _ = f.snapshot() + row := f.row(0) + segment := row.Segments()[0] + bitmap := segment.data + + // request information from the frozen bitmap we got back + go func() { + for atomic.LoadInt64(&done) == 0 { + for i := 0; i < f.MaxOpN; i++ { + _ = bitmap.Contains(uint64(i * 32)) + } + } + close(ch) + }() + + // modify the original bitmap, until it causes a snapshot, which + // then invalidates the other map... + for j := 0; j < 5; j++ { + for i := 0; i < f.MaxOpN; i++ { + _, _ = f.setBit(0, uint64(i*32+j+1)) + } + } + atomic.StoreInt64(&done, 1) + <-ch +} + // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") @@ -137,14 +178,14 @@ func TestFragment_SetRow(t *testing.T) { rowID := uint64(1000) // Set bits on the fragment. - if _, err := f.setBit(rowID, 8000001); err != nil { + if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil { t.Fatal(err) - } else if _, err := f.setBit(rowID, 8065536); err != nil { + } else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil { t.Fatal(err) } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) { t.Fatalf("unexpected columns: %+v", cols) } // Verify count on row. @@ -153,7 +194,7 @@ func TestFragment_SetRow(t *testing.T) { } // Set row (overwrite existing data). - row := NewRow(8000002, 8065537, 8131074) + row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000) if changed, err := f.unprotectedSetRow(row, rowID); err != nil { t.Fatal(err) } else if !changed { @@ -161,7 +202,7 @@ func TestFragment_SetRow(t *testing.T) { } // Verify data on row. - if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) { + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) { t.Fatalf("unexpected columns after set row: %+v", cols) } // Verify count on row. @@ -275,7 +316,7 @@ func TestFragment_SetValue(t *testing.T) { } // Non-existent value. - if value, exists, err := f.value(100, 11); err != nil { + if value, exists, err := f.value(101, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -304,7 +345,7 @@ func TestFragment_SetValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.setValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, int64(value)); err != nil { t.Fatal(err) } } @@ -408,7 +449,7 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Min", func(t *testing.T) { tests := []struct { filter *Row - exp uint64 + exp int64 cnt uint64 }{ {filter: nil, exp: 0, cnt: 1}, @@ -432,7 +473,7 @@ func TestFragment_MinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { tests := []struct { filter *Row - exp uint64 + exp int64 cnt uint64 }{ {filter: nil, exp: 2818, cnt: 2}, @@ -443,12 +484,15 @@ func TestFragment_MinMax(t *testing.T) { {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { + var columns []uint64 + if test.filter != nil { + columns = test.filter.Columns() + } + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) - } else if max != test.exp { - t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) - } else if cnt != test.cnt { - t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } else if max != test.exp || cnt != test.cnt { + t.Errorf("%d. max(%v, %v)=(%v, %v), expected (%v, %v)", i, columns, bitDepth, max, cnt, test.exp, test.cnt) } } }) @@ -654,7 +698,9 @@ func TestFragment_Range(t *testing.T) { func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { column := uint64(0) for i := 0; i < b.N; i++ { - f.setValue(column, bitDepth, uint64(i)) + // We're not checking the error because this is a benchmark. + // That does mean the result could be completely wrong... + _, _ = f.setValue(column, bitDepth, int64(i)) column = cfunc(column) } } @@ -683,9 +729,9 @@ func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func( column := uint64(0) b.StopTimer() columns := make([]uint64, b.N) - values := make([]uint64, b.N) + values := make([]int64, b.N) for i := 0; i < b.N; i++ { - values[i] = uint64(i) + values[i] = int64(i) columns[i] = column column = cfunc(column) } @@ -701,12 +747,12 @@ func BenchmarkFragment_ImportValue(b *testing.B) { depths := []uint{4, 8, 16} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) b.Run(name+"_Sparse", func(b *testing.B) { benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) f.Clean(b) - f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f = mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) b.Run(name+"_Dense", func(b *testing.B) { benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) @@ -743,7 +789,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) - err := f.importRoaring(getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) + err := f.importRoaringT(getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) } @@ -779,14 +825,14 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") f.MaxOpN = opN defer f.Clean(b) - err := f.importRoaring(getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) + err := f.importRoaringT(getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) } for i := 0; i < numUpdates; i++ { data := getUpdataRoaring(numRows, bitsPerUpdate, int64(i)) b.StartTimer() - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) b.StopTimer() if err != nil { b.Fatalf("doing small roaring import: %v", err) @@ -802,30 +848,30 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { initialCols := make([]uint64, 0, ShardWidth) - initialVals := make([]uint64, 0, ShardWidth) + initialVals := make([]int64, 0, ShardWidth) for i := uint64(0); i < ShardWidth; i++ { // every 29 columns, skip between 0 and 12 columns if i%29 == 0 { i += i % 13 } initialCols = append(initialCols, i) - initialVals = append(initialVals, uint64(rand.Int63n(1<<21))) + initialVals = append(initialVals, int64(rand.Int63n(1<<21))) } for _, numUpdates := range []int{100} { for _, valsPerUpdate := range []int{10, 100} { updateCols := make([]uint64, numUpdates*valsPerUpdate) - updateVals := make([]uint64, numUpdates*valsPerUpdate) + updateVals := make([]int64, numUpdates*valsPerUpdate) for i := 0; i < numUpdates*valsPerUpdate; i++ { updateCols[i] = uint64(rand.Int63n(ShardWidth)) - updateVals[i] = uint64(rand.Int63n(1 << 21)) + updateVals[i] = int64(rand.Int63n(1 << 21)) } for _, opN := range []int{1, 5000, 50000} { b.Run(fmt.Sprintf("Updates%dVals%dOpN%d", numUpdates, valsPerUpdate, opN), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, CacheTypeNone) + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = opN err := f.importValue(initialCols, initialVals, 21, false) if err != nil { @@ -943,8 +989,14 @@ func TestFragment_Top_Filter(t *testing.T) { f.mustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. - f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) - f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) + err := f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) + if err != nil { + t.Fatalf("setAttrs: %v", err) + } + err = f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) + if err != nil { + t.Fatalf("setAttrs: %v", err) + } // Retrieve top rows. if pairs, err := f.top(topOptions{ @@ -1016,7 +1068,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { if err != nil { t.Fatalf("writing to bytes: %v", err) } - err = f.importRoaring(b.Bytes(), false) + err = f.importRoaringT(b.Bytes(), false) if err != nil { t.Fatalf("importing data: %v", err) } @@ -1363,7 +1415,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1892,7 +1944,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1914,7 +1966,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(b) // Generate some intersecting data. - maxX := 1048576 / 2 + maxX := ShardWidth / 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) @@ -1950,7 +2002,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { b.StopTimer() - maxX := 1048576 * 5 * 2 + maxX := ShardWidth * 5 * 2 sz := maxX rows := make([]uint64, sz) cols := make([]uint64, sz) @@ -2000,7 +2052,7 @@ func BenchmarkImportRoaring(b *testing.B) { for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) b.StartTimer() - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { f.Clean(b) b.Fatalf("import error: %v", err) @@ -2033,7 +2085,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - return frags[j].importRoaring(data, false) + return frags[j].importRoaringT(data, false) }) } err := eg.Wait() @@ -2064,14 +2116,17 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) - frags[j].importRoaring(data, false) + err := frags[j].importRoaringT(data, false) + if err != nil { + b.Fatalf("importing roaring: %v", err) + } } eg := errgroup.Group{} b.StartTimer() for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - return frags[j].importRoaring(updata, false) + return frags[j].importRoaringT(updata, false) }) } err := eg.Wait() @@ -2127,12 +2182,12 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { b.Errorf("import error: %v", err) } b.StartTimer() - err = f.importRoaring(updata, false) + err = f.importRoaringT(updata, false) if err != nil { f.Clean(b) b.Errorf("import error: %v", err) @@ -2163,12 +2218,12 @@ func BenchmarkUpdatePathological(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) - err := f.importRoaring(exists, false) + err := f.importRoaringT(exists, false) if err != nil { b.Fatalf("importing roaring: %v", err) } b.StartTimer() - err = f.importRoaring(inc, false) + err = f.importRoaringT(inc, false) if err != nil { b.Fatalf("importing second: %v", err) } @@ -2185,7 +2240,7 @@ func initBigFrag() { for i := int64(0); i < 10; i++ { // 10 million rows, 1 bit per column, random seeded by i data := getZipfRowsSliceRoaring(10000000, i, 0, ShardWidth) - err := f.importRoaring(data, false) + err := f.importRoaringT(data, false) if err != nil { panic(fmt.Sprintf("setting up fragment data: %v", err)) } @@ -2219,7 +2274,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - nf := newFragment(fi.Name(), "i", "f", viewStandard, 0) + nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) @@ -2256,13 +2311,13 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - nf := newFragment(fi.Name(), "i", "f", viewStandard, 0) + nf := newFragment(fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) } b.StartTimer() - err = nf.importRoaring(updata, false) + err = nf.importRoaringT(updata, false) b.StopTimer() if err != nil { b.Fatalf("bulkImport: %v", err) @@ -2275,7 +2330,10 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { func TestGetZipfRowsSliceRoaring(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) - f.importRoaring(data, false) + err := f.importRoaringT(data, false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { t.Fatalf("unexpected rows: %v", f.rows(0)) } @@ -2417,6 +2475,11 @@ func (f *fragment) Clean(t testing.TB) { } } +// importRoaringT calls importRoaring with context.Background() for convenience +func (f *fragment) importRoaringT(data []byte, clear bool) error { + return f.importRoaring(context.Background(), data, clear) +} + // CleanKeep is just like Clean(), but it doesn't remove the // fragment file (note that it DOES remove the cache file). func (f *fragment) CleanKeep(t testing.TB) { @@ -2433,6 +2496,15 @@ func (f *fragment) CleanKeep(t testing.TB) { // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { + return mustOpenFragmentFlags(index, field, view, shard, cacheType, 0) +} + +func mustOpenBSIFragment(index, field, view string, shard uint64) *fragment { + return mustOpenFragmentFlags(index, field, view, shard, "", 1) +} + +// mustOpenFragment returns a new instance of Fragment with a temporary path. +func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) *fragment { file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") if err != nil { panic(err) @@ -2443,7 +2515,7 @@ func mustOpenFragment(index, field, view string, shard uint64, cacheType string) cacheType = DefaultCacheType } - f := newFragment(file.Name(), index, field, view, shard) + f := newFragment(file.Name(), index, field, view, shard, flags) f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ store: make(map[uint64]map[string]interface{}), @@ -2608,7 +2680,10 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaring(buf.Bytes(), false) + err = f.importRoaringT(buf.Bytes(), false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } exp := calcExpected(test[:num+1]...) for row, expCols := range exp { cols := f.row(uint64(row)).Columns() @@ -2680,7 +2755,10 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaring(buf.Bytes(), false) + err = f.importRoaringT(buf.Bytes(), false) + if err != nil { + t.Fatalf("importing roaring: %v", err) + } rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) pairs, err = f.top(topOptions{}) @@ -2737,7 +2815,7 @@ func calcTop(rowIDs, colIDs []uint64) []Pair { func calcExpected(inputs ...[]uint64) [][]uint64 { // create map of row id to set of column values in that row. rows := make(map[uint64]map[uint64]struct{}) - var maxrow uint64 = 0 + var maxrow uint64 for _, input := range inputs { for _, val := range input { row := val / ShardWidth @@ -2895,19 +2973,56 @@ func TestFragmentRowIterator(t *testing.T) { func TestUnionInPlaceMapped(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) defer f.Clean(t) + // I know this doesn't actually matter in our current context, but + // strictly speaking, we do say you have to hold the lock while calling + // unprotectedWriteToFragment... + f.mu.Lock() + defer f.mu.Unlock() r0 := rand.New(rand.NewSource(2)) r1 := rand.New(rand.NewSource(1)) data0 := randPositions(1000000, r0) - setBM := roaring.NewBitmap() - setBM.OpWriter = nil - setBM.Add(data0...) - unprotectedWriteToFragment(f, setBM) - data1 := randPositions(1000000, r1) - setBM2 := roaring.NewBitmap() - setBM2.OpWriter = nil - setBM2.Add(data1...) + setBM0 := roaring.NewBitmap() + setBM0.OpWriter = nil + _, err := setBM0.Add(data0...) + if err != nil { + t.Fatalf("adding bits: %v", err) + } + count0 := setBM0.Count() - f.storage.UnionInPlace(setBM2) + data1 := randPositions(1000000, r1) + setBM1 := roaring.NewBitmap() + setBM1.OpWriter = nil + _, err = setBM1.Add(data1...) + if err != nil { + t.Fatalf("adding bits: %v", err) + } + count1 := setBM1.Count() + + // now we write setBM0 into f.storage. + _, err = unprotectedWriteToFragment(f, setBM0) + if err != nil { + t.Fatalf("trying to flush fragment to disk: %v", err) + } + countF := f.storage.Count() + + f.storage.UnionInPlace(setBM1) + countUnion := f.storage.Count() + + if count0 != countF { + t.Fatalf("writing bitmap to storage changed count: %d => %d", count0, countF) + } + min := count0 + if count1 > min { + min = count1 + } + max := count0 + count1 + // We don't know how many bits we should have, because of overlap, + // but it should be between the size of the largest bitmap and the + // sum of the bitmaps. + if countUnion < min || countUnion > max { + t.Fatalf("union of sets with cardinality %d and %d should be between %d and %d, got %d", + count0, count1, min, max, countUnion) + } } func randPositions(n int, r *rand.Rand) []uint64 { @@ -2925,7 +3040,7 @@ func TestFragmentPositionsForValue(t *testing.T) { tests := []struct { columnID uint64 bitDepth uint - value uint64 + value int64 clear bool toSet []uint64 toClear []uint64 @@ -2934,43 +3049,43 @@ func TestFragmentPositionsForValue(t *testing.T) { columnID: 0, bitDepth: 1, value: 0, - toSet: []uint64{ShardWidth}, - toClear: []uint64{0}, + toSet: []uint64{0}, // exists bit only + toClear: []uint64{ShardWidth, ShardWidth * 2}, // sign bit & 1-position }, { columnID: 0, bitDepth: 3, value: 0, - toSet: []uint64{ShardWidth * 3}, - toClear: []uint64{0, ShardWidth, ShardWidth * 2}, + toSet: []uint64{0}, // exists bit only + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 3, ShardWidth * 4}, // sign bit, 1, 2, 4 }, { columnID: 1, bitDepth: 3, value: 0, - toSet: []uint64{ShardWidth*3 + 1}, - toClear: []uint64{1, ShardWidth + 1, ShardWidth*2 + 1}, + toSet: []uint64{1}, // exists bit only + toClear: []uint64{ShardWidth + 1, ShardWidth*2 + 1, ShardWidth*3 + 1, ShardWidth*4 + 1}, // sign bit, 1, 2, 4 }, { columnID: 0, bitDepth: 1, value: 1, - toSet: []uint64{0, ShardWidth}, - toClear: []uint64{}, + toSet: []uint64{0, ShardWidth * 2}, // exists bit, 1 + toClear: []uint64{ShardWidth}, // sign bit only }, { columnID: 0, bitDepth: 4, value: 10, - toSet: []uint64{ShardWidth, ShardWidth * 3, ShardWidth * 4}, - toClear: []uint64{0, ShardWidth * 2}, + toSet: []uint64{0, ShardWidth * 3, ShardWidth * 5}, // exists bit, 2, 8 + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 4}, // sign bit, 1, 4 }, { columnID: 0, bitDepth: 5, value: 10, - toSet: []uint64{ShardWidth, ShardWidth * 3, ShardWidth * 5}, - toClear: []uint64{0, ShardWidth * 2, ShardWidth * 4}, + toSet: []uint64{0, ShardWidth * 3, ShardWidth * 5}, // exists bit, 2, 8 + toClear: []uint64{ShardWidth * 1, ShardWidth * 2, ShardWidth * 4, ShardWidth * 6}, // sign bit, 1, 4, 16 }, } @@ -3075,7 +3190,7 @@ func TestImportClearRestart(t *testing.T) { check(t, f, exp) - f2 := newFragment(f.path, "i", "f", viewStandard, 0) + f2 := newFragment(f.path, "i", "f", viewStandard, 0, 0) f2.MaxOpN = maxOpN f2.CacheType = f.CacheType @@ -3109,7 +3224,7 @@ func TestImportClearRestart(t *testing.T) { check(t, f2, exp) - f3 := newFragment(f2.path, "i", "f", viewStandard, 0) + f3 := newFragment(f2.path, "i", "f", viewStandard, 0, 0) f3.MaxOpN = maxOpN f3.CacheType = f.CacheType @@ -3160,13 +3275,13 @@ func check(t *testing.T, f *fragment, exp map[uint64]map[uint64]struct{}) { } func TestImportValueConcurrent(t *testing.T) { - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i eg.Go(func() error { for j := uint64(0); j < 10; j++ { - err := f.importValue([]uint64{j}, []uint64{uint64(rand.Int63n(1000))}, 10, i%2 == 0) + err := f.importValue([]uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) if err != nil { return err } @@ -3183,14 +3298,14 @@ func TestImportValueConcurrent(t *testing.T) { func TestImportMultipleValues(t *testing.T) { tests := []struct { cols []uint64 - vals []uint64 + vals []int64 checkCols []uint64 checkVals []uint64 depth uint }{ { cols: []uint64{0, 0}, - vals: []uint64{97, 100}, + vals: []int64{97, 100}, depth: 7, checkCols: []uint64{0}, checkVals: []uint64{100}, @@ -3200,7 +3315,7 @@ func TestImportMultipleValues(t *testing.T) { for i, test := range tests { for _, maxOpN := range []int{0, 10000} { // test small/large write t.Run(fmt.Sprintf("%dLowOpN", i), func(t *testing.T) { - f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, CacheTypeNone) + f := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = maxOpN defer f.Clean(t) err := f.importValue(test.cols, test.vals, test.depth, false) @@ -3226,3 +3341,30 @@ func TestImportMultipleValues(t *testing.T) { } } } + +func TestFragmentConcurrentReadWrite(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + defer f.Clean(t) + + eg := &errgroup.Group{} + eg.Go(func() error { + for i := uint64(0); i < 1000; i++ { + _, err := f.setBit(i%4, i) + if err != nil { + return errors.Wrap(err, "setting bit") + } + } + return nil + }) + + acc := uint64(0) + for i := uint64(0); i < 100; i++ { + r := f.row(i % 4) + acc += r.Count() + } + if err := eg.Wait(); err != nil { + t.Errorf("error from setting a bit: %v", err) + } + + t.Logf("%d", acc) +} diff --git a/gc.go b/gc.go index 1456c22c2..e5e5d7031 100644 --- a/gc.go +++ b/gc.go @@ -32,6 +32,6 @@ type nopGCNotifier struct{} func (n *nopGCNotifier) Close() {} // AfterGC is a no-op implementation of GCNotifier AfterGC method. -func (c *nopGCNotifier) AfterGC() <-chan struct{} { +func (n *nopGCNotifier) AfterGC() <-chan struct{} { return nil } diff --git a/go.mod b/go.mod index 2cfa54cd8..397d0e267 100644 --- a/go.mod +++ b/go.mod @@ -1,32 +1,42 @@ module github.com/pilosa/pilosa -replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 +replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 require ( + github.com/BurntSushi/toml v0.3.1 // indirect github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.2.0 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/opentracing/opentracing-go v1.0.2 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 - github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 - golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 + github.com/uber/jaeger-client-go v2.16.0+incompatible + github.com/uber/jaeger-lib v2.0.0+incompatible // indirect + golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect + golang.org/x/text v0.3.2 // indirect + golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 ) diff --git a/go.sum b/go.sum index e77168275..674e2adea 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,13 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -19,6 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -29,8 +36,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= -github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -43,6 +48,7 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -50,6 +56,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -58,17 +66,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo= -github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= +github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -78,6 +84,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -91,27 +99,53 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= +github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU= +golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1 h1:MSSXVSCgrxTAYytvleklMKlLdxjexiJWNffJciO1nCI= +golang.org/x/tools v0.0.0-20190429231329-9d4d845e86f1/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index b50885241..13ada756e 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -139,9 +139,7 @@ func (s *systemInfo) collectPlatformInfo() error { // we have no way to know, let's try runtime s.cpuLogicalCores = runtime.NumCPU() } - if err != nil { - return err - } + return nil } return nil } diff --git a/handler.go b/handler.go index 5a452ce96..f42cf1993 100644 --- a/handler.go +++ b/handler.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -60,6 +74,8 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { }) } +// Handler is the interface for the data handler, a wrapper around +// Pilosa's data store. type Handler interface { Serve() error Close() error @@ -75,8 +91,11 @@ func (n nopHandler) Close() error { return nil } +// NopHandler is a no-op implementation of the Handler interface. var NopHandler Handler = nopHandler{} +// ImportValueRequest describes the import request structure +// for a value (BSI) import. type ImportValueRequest struct { Index string Field string @@ -86,6 +105,8 @@ type ImportValueRequest struct { Values []int64 } +// ImportRequest describes the import request structure +// for an import. type ImportRequest struct { Index string Field string @@ -97,15 +118,20 @@ type ImportRequest struct { Timestamps []int64 } +// ImportRoaringRequest describes the import request structure +// for an import containing roaring-encoded data. type ImportRoaringRequest struct { Clear bool Views map[string][]byte } +// ImportResponse is the structured response of an import. type ImportResponse struct { Err string } +// BlockDataRequest describes the structure of a request +// for fragment block data. type BlockDataRequest struct { Index string Field string @@ -114,17 +140,23 @@ type BlockDataRequest struct { Block uint64 } +// BlockDataResponse is the structured response of a block +// data request. type BlockDataResponse struct { RowIDs []uint64 ColumnIDs []uint64 } +// TranslateKeysRequest describes the structure of a request +// for a batch of key translations. type TranslateKeysRequest struct { Index string Field string Keys []string } +// TranslateKeysResponse is the structured response of a key +// translation request. type TranslateKeysResponse struct { IDs []uint64 } diff --git a/holder.go b/holder.go index d751c90b2..9384d6622 100644 --- a/holder.go +++ b/holder.go @@ -77,8 +77,6 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. cacheFlushInterval time.Duration - shardValidatorFunc func(index string, shard uint64) bool - Logger logger.Logger } @@ -125,9 +123,6 @@ func NewHolder() *Holder { NewAttrStore: newNopAttrStore, cacheFlushInterval: defaultCacheFlushInterval, - shardValidatorFunc: func(index string, shard uint64) bool { - return true //default - }, Logger: logger.NopLogger, } @@ -430,9 +425,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { index.broadcaster = h.broadcaster index.newAttrStore = h.NewAttrStore index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data")) - index.shardValidator = func(shard uint64) bool { - return h.shardValidatorFunc(name, shard) - } return index, nil } @@ -587,7 +579,6 @@ func (h *Holder) setFileLimit() { func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, ".id") - nodeID := "" h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", errors.Wrap(err, "creating directory") @@ -595,17 +586,16 @@ func (h *Holder) loadNodeID() (string, error) { nodeIDBytes, err := ioutil.ReadFile(idPath) if err == nil { - nodeID = strings.TrimSpace(string(nodeIDBytes)) - } else if os.IsNotExist(err) { - nodeID = uuid.NewV4().String() - err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) - if err != nil { - return "", errors.Wrap(err, "writing file") - } - } else if err != nil { + return strings.TrimSpace(string(nodeIDBytes)), nil + } + if !os.IsNotExist(err) { return "", errors.Wrap(err, "reading file") } - + nodeID := uuid.NewV4().String() + err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) + if err != nil { + return "", errors.Wrap(err, "writing file") + } return nodeID, nil } diff --git a/holder_internal_test.go b/holder_internal_test.go index b13df7809..0c96a81ea 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -112,8 +112,10 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) - + defer func() { + // we don't care about a failure here + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -136,7 +138,10 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + defer func() { + // we don't care about a failure here + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) @@ -165,8 +170,9 @@ func TestHolder_Optn(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -209,8 +215,14 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) // Set highest shard. - hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1)) - hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2)) + err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1)) + if err != nil { + t.Fatalf("adding remote shards: %v", err) + } + err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2)) + if err != nil { + t.Fatalf("adding remote shards: %v", err) + } // Keep replication the same and ensure we get the expected results. cluster.ReplicaN = 2 @@ -292,8 +304,20 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { func TestHolderCleaner_Reopen(t *testing.T) { h := NewHolder() h.Path = "path" - h.Open() - h.Close() - h.Open() - h.Close() + err := h.Open() + if err != nil { + t.Fatalf("couldn't open holder: %v", err) + } + err = h.Close() + if err != nil { + t.Fatalf("couldn't close holder: %v", err) + } + err = h.Open() + if err != nil { + t.Fatalf("couldn't open holder: %v", err) + } + err = h.Close() + if err != nil { + t.Fatalf("couldn't close holder: %v", err) + } } diff --git a/holder_test.go b/holder_test.go index 1bfe4615a..525ca1dc7 100644 --- a/holder_test.go +++ b/holder_test.go @@ -67,7 +67,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(h.IndexPath("test"), 0777) + defer func() { + _ = os.Chmod(h.IndexPath("test"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) @@ -106,8 +108,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0777) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0755) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } @@ -167,8 +170,9 @@ func TestHolder_Open(t *testing.T) { } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { t.Fatal(err) } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0666) - + defer func() { + _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0644) + }() if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { t.Fatalf("unexpected error: %s", err) } diff --git a/http/client.go b/http/client.go index 92f6269fe..74706588d 100644 --- a/http/client.go +++ b/http/client.go @@ -132,6 +132,33 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error return rsp.Indexes, nil } +func (c *InternalClient) PostSchema(ctx context.Context, uri *pilosa.URI, s *pilosa.Schema, remote bool) error { + u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote)) + buf, err := json.Marshal(s) + if err != nil { + return errors.Wrap(err, "marshalling schema") + } + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + return errors.Errorf("unexpected status code: %s", resp.Status) + } + return nil +} + // CreateIndex creates a new index on the server. func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex") @@ -629,7 +656,11 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind dec := json.NewDecoder(resp.Body) rbody := &pilosa.ImportResponse{} - dec.Decode(rbody) + err = dec.Decode(rbody) + // Decode can return EOF when no error occurred. helpful! + if err != nil && err != io.EOF { + return errors.Wrap(err, "decoding response body") + } if rbody.Err != "" { return errors.Wrap(errors.New(rbody.Err), "importing roaring") } diff --git a/http/client_test.go b/http/client_test.go index 2b65f82d4..b4c32932d 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -116,9 +116,18 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - c[0].RecalculateCaches() - c[1].RecalculateCaches() - c[2].RecalculateCaches() + err = c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } + err = c[1].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } + err = c[2].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating cache: %v", err) + } // Connect to each node to compare results. client := make([]*Client, 3) @@ -173,7 +182,10 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can export data. func TestClient_Export(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] + host := cmd.URL() cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true}) @@ -336,7 +348,9 @@ func TestClient_Export(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -505,7 +519,9 @@ func TestClient_ImportRoaring(t *testing.T) { // Ensure client can bulk import data. func TestClient_ImportKeys(t *testing.T) { t.Run("SingleNode", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true}) @@ -602,6 +618,7 @@ func TestClient_ImportKeys(t *testing.T) { t.Run("MultiNode", func(t *testing.T) { cluster := test.MustRunCluster(t, 2) + defer cluster.Close() cmd0 := cluster[0] cmd1 := cluster[1] host0 := cmd0.URL() @@ -677,7 +694,9 @@ func TestClient_ImportKeys(t *testing.T) { }) t.Run("IntegerFieldSingleNode", func(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -760,7 +779,9 @@ func TestClient_ImportKeys(t *testing.T) { // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -811,8 +832,8 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - if min != -100 || cnt != 0 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + if min != 0 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt) } // Verify Max. @@ -850,8 +871,8 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - if min != -100 || cnt != 0 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + if min != 0 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=0, cnt=0", min, cnt) } // Verify Max. @@ -866,7 +887,9 @@ func TestClient_ImportValue(t *testing.T) { // Ensure client can bulk import data while tracking existence. func TestClient_ImportExistence(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -943,7 +966,10 @@ func TestClient_ImportExistence(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] + holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/http/handler.go b/http/handler.go index 3226eee50..8e932271a 100644 --- a/http/handler.go +++ b/http/handler.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "io/ioutil" + "math" "net" "net/http" _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. @@ -187,6 +188,7 @@ func (h *Handler) populateValidators() { h.validators["GetInfo"] = queryValidationSpecRequired() h.validators["RecalculateCaches"] = queryValidationSpecRequired() h.validators["GetSchema"] = queryValidationSpecRequired() + h.validators["PostSchema"] = queryValidationSpecRequired().Optional("remote") h.validators["GetStatus"] = queryValidationSpecRequired() h.validators["GetVersion"] = queryValidationSpecRequired() h.validators["PostClusterMessage"] = queryValidationSpecRequired() @@ -255,6 +257,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") @@ -320,6 +323,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { + h *Handler Success bool `json:"success"` Error *Error `json:"error,omitempty"` } @@ -347,7 +351,7 @@ func (r *successResponse) check(err error) (statusCode int) { } r.Success = false - r.Error = &Error{Message: cause.Error()} + r.Error = &Error{Message: err.Error()} return statusCode } @@ -367,8 +371,18 @@ func (r *successResponse) write(w http.ResponseWriter, err error) { // Write the response. if statusCode == 0 { - w.Write(msg) - w.Write([]byte("\n")) + _, err := w.Write(msg) + if err != nil { + r.h.logger.Printf("error writing response: %v", err) + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + _, err = w.Write([]byte("\n")) + if err != nil { + r.h.logger.Printf("error writing newline after response: %v", err) + http.Error(w, string(msg), http.StatusInternalServerError) + return + } } else { http.Error(w, string(msg), statusCode) } @@ -401,11 +415,32 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { + if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? h.logger.Printf("write schema response error: %s", err) } } +func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + remoteStr := q.Get("remote") + var remote bool + if remoteStr == "true" { + remote = true + } + + schema := &pilosa.Schema{} + if err := json.NewDecoder(r.Body).Decode(schema); err != nil { + http.Error(w, fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err), http.StatusBadRequest) + return + } + + if err := h.api.ApplySchema(r.Context(), schema, remote); err != nil { + http.Error(w, fmt.Sprintf("apply schema to Pilosa: %v", err), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) +} + // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { @@ -449,7 +484,10 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { req, err := h.readQueryRequest(r) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + if e != nil { + h.logger.Printf("write query response error: %v (while trying to write another error: %v)", e, err) + } return } // TODO: Remove @@ -463,7 +501,10 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { default: w.WriteHeader(http.StatusBadRequest) } - h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + e := h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + if e != nil { + h.logger.Printf("write query response error: %v (while trying to write another error: %v)", e, err) + } return } @@ -612,7 +653,7 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteIndex(r.Context(), indexName) resp.write(w, err) } @@ -625,7 +666,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { } indexName := mux.Vars(r)["index"] - resp := successResponse{} + resp := successResponse{h: h} // Decode request. req := postIndexRequest{ @@ -694,7 +735,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] - resp := successResponse{} + resp := successResponse{h: h} // Decode request. var req postFieldRequest @@ -718,6 +759,14 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { case pilosa.FieldTypeSet: fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: + if req.Options.Min == nil { + min := int64(math.MinInt64) + req.Options.Min = &min + } + if req.Options.Max == nil { + max := int64(math.MaxInt64) + req.Options.Max = &max + } fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) case pilosa.FieldTypeTime: fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) @@ -733,6 +782,10 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } resp.write(w, err) } @@ -784,10 +837,6 @@ func (o *fieldOptions) validate() error { return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) } else if o.CacheSize != nil { return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) - } else if o.Min == nil { - return pilosa.NewBadRequestError(errors.New("min is required for field type int")) - } else if o.Max == nil { - return pilosa.NewBadRequestError(errors.New("max is required for field type int")) } else if o.TimeQuantum != nil { return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) } @@ -847,7 +896,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteField(r.Context(), indexName, fieldName) resp.write(w, err) } @@ -863,7 +912,7 @@ func (h *Handler) handleDeleteRemoteAvailableShard(w http.ResponseWriter, r *htt fieldName := mux.Vars(r)["field"] shardID, _ := strconv.ParseUint(mux.Vars(r)["shardID"], 10, 64) - resp := successResponse{} + resp := successResponse{h: h} err := h.api.DeleteAvailableShard(r.Context(), indexName, fieldName, shardID) resp.write(w, err) } @@ -1080,7 +1129,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Write response. - w.Write(buf) + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } } // handleGetExport handles /export requests. @@ -1179,7 +1231,10 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Write response. w.Header().Set("Content-Type", "application/protobuf") w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing fragment/block/data response: %v", err) + } } // handleGetFragmentBlocks handles GET /internal/fragment/blocks requests. @@ -1571,15 +1626,23 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request remote = true } + ctx := r.Context() + // Read entire body. + span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") body, err := ioutil.ReadAll(r.Body) + span.LogKV("bodySize", len(body)) + span.Finish() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } req := &pilosa.ImportRoaringRequest{} - if err := h.api.Serializer.Unmarshal(body, req); err != nil { + span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") + err = h.api.Serializer.Unmarshal(body, req) + span.Finish() + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -1593,7 +1656,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, remote, req) + err = h.api.ImportRoaring(ctx, indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { @@ -1602,6 +1665,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request w.WriteHeader(http.StatusInternalServerError) } } + // Marshal response object. buf, err := h.api.Serializer.Marshal(resp) if err != nil { diff --git a/http/translator.go b/http/translator.go index 3a2acae04..251715ede 100644 --- a/http/translator.go +++ b/http/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package http import ( diff --git a/http/translator_test.go b/http/translator_test.go index ce96b3c2c..87bb4366d 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package http_test import ( @@ -23,7 +37,9 @@ func TestTranslateStore_Reader(t *testing.T) { // "translator_test.go:65: unexpected EOF" t.Skip() - primary := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + primary := cluster[0] hldr := test.Holder{Holder: primary.Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) @@ -89,9 +105,10 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts}) + defer cluster.Close() + primary := cluster[0] - defer primary.Close() defer close(done) // Connect to server and begin streaming. @@ -121,8 +138,9 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] - defer primary.Close() + cluster := test.MustRunCluster(t, 1, []server.CommandOption{opts}) + defer cluster.Close() + primary := cluster[0] ts := http.NewTranslateStore(primary.URL()) _, err := ts.Reader(context.Background(), 0) diff --git a/index.go b/index.go index e94692288..dbb9c052a 100644 --- a/index.go +++ b/index.go @@ -50,9 +50,8 @@ type Index struct { // Column attribute storage and cache. columnAttrs AttrStore - broadcaster broadcaster - Stats stats.StatsClient - shardValidator func(uint64) bool + broadcaster broadcaster + Stats stats.StatsClient logger logger.Logger } @@ -75,7 +74,6 @@ func NewIndex(path, name string) (*Index, error) { broadcaster: NopBroadcaster, Stats: stats.NopStatsClient, logger: logger.NopLogger, - shardValidator: defaultShardValidator, trackExistence: true, }, nil } @@ -109,15 +107,18 @@ func (i *Index) options() IndexOptions { // Open opens and initializes the index. func (i *Index) Open() error { // Ensure the path exists. + i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { return errors.Wrap(err, "creating directory") } // Read meta file. + i.logger.Debugf("load meta file for index: %s", i.name) if err := i.loadMeta(); err != nil { return errors.Wrap(err, "loading meta file") } + i.logger.Debugf("open fields for index: %s", i.name) if err := i.openFields(); err != nil { return errors.Wrap(err, "opening fields") } @@ -153,13 +154,15 @@ func (i *Index) openFields() error { continue } + i.logger.Debugf("open field: %s", fi.Name()) fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { - return ErrName + return errors.Wrapf(ErrName, "'%s'", fi.Name()) } if err := fld.Open(); err != nil { return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } + i.logger.Debugf("add field to index.fields: %s", fi.Name()) i.fields[fld.Name()] = fld } return nil @@ -405,7 +408,6 @@ func (i *Index) newField(path, name string) (*Field, error) { f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) - f.shardValidator = i.shardValidator return f, nil } diff --git a/index_test.go b/index_test.go index 21f9a91d9..ee5625cf3 100644 --- a/index_test.go +++ b/index_test.go @@ -93,7 +93,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with schema and verify it exists. - if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil { + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(-990, 1000)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) diff --git a/inmem/translator.go b/inmem/translator.go index 4e1e96b55..c5f976ea2 100644 --- a/inmem/translator.go +++ b/inmem/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package inmem import ( diff --git a/inmem/translator_test.go b/inmem/translator_test.go index d4d232566..0a7c6eb34 100644 --- a/inmem/translator_test.go +++ b/inmem/translator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package inmem_test import ( diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index f01835c10..612189c1b 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clustertest import ( diff --git a/internal/private.pb.go b/internal/private.pb.go index c5a51741b..fc6c306e9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,48 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto +/* + Package internal is a generated protocol buffer package. + + It is generated from these files: + private.proto + + It has these top-level messages: + IndexMeta + FieldOptions + ImportResponse + BlockDataRequest + BlockDataResponse + Cache + MaxShards + CreateShardMessage + DeleteIndexMessage + CreateIndexMessage + CreateFieldMessage + DeleteFieldMessage + DeleteAvailableShardMessage + Field + Schema + Index + URI + Node + NodeStateMessage + NodeEventMessage + NodeStatus + IndexStatus + FieldStatus + ClusterStatus + BSIGroup + CreateViewMessage + DeleteViewMessage + ResizeInstruction + ResizeSource + ResizeInstructionComplete + SetCoordinatorMessage + UpdateCoordinatorMessage + Topology + RecalculateCaches +*/ package internal import proto "github.com/golang/protobuf/proto" @@ -21,45 +63,14 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` - TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` + TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` } -func (m *IndexMeta) Reset() { *m = IndexMeta{} } -func (m *IndexMeta) String() string { return proto.CompactTextString(m) } -func (*IndexMeta) ProtoMessage() {} -func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{0} -} -func (m *IndexMeta) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *IndexMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_IndexMeta.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *IndexMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexMeta.Merge(dst, src) -} -func (m *IndexMeta) XXX_Size() int { - return m.Size() -} -func (m *IndexMeta) XXX_DiscardUnknown() { - xxx_messageInfo_IndexMeta.DiscardUnknown(m) -} - -var xxx_messageInfo_IndexMeta proto.InternalMessageInfo +func (m *IndexMeta) Reset() { *m = IndexMeta{} } +func (m *IndexMeta) String() string { return proto.CompactTextString(m) } +func (*IndexMeta) ProtoMessage() {} +func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } func (m *IndexMeta) GetKeys() bool { if m != nil { @@ -76,51 +87,22 @@ func (m *IndexMeta) GetTrackExistence() bool { } type FieldOptions struct { - Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` - Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` - NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` + NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"` + Base int64 `protobuf:"varint,13,opt,name=Base,proto3" json:"Base,omitempty"` + BitDepth uint64 `protobuf:"varint,14,opt,name=BitDepth,proto3" json:"BitDepth,omitempty"` + Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *FieldOptions) Reset() { *m = FieldOptions{} } -func (m *FieldOptions) String() string { return proto.CompactTextString(m) } -func (*FieldOptions) ProtoMessage() {} -func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{1} -} -func (m *FieldOptions) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldOptions.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldOptions) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldOptions.Merge(dst, src) -} -func (m *FieldOptions) XXX_Size() int { - return m.Size() -} -func (m *FieldOptions) XXX_DiscardUnknown() { - xxx_messageInfo_FieldOptions.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldOptions proto.InternalMessageInfo +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } func (m *FieldOptions) GetType() string { if m != nil { @@ -143,20 +125,6 @@ func (m *FieldOptions) GetCacheSize() uint32 { return 0 } -func (m *FieldOptions) GetMin() int64 { - if m != nil { - return m.Min - } - return 0 -} - -func (m *FieldOptions) GetMax() int64 { - if m != nil { - return m.Max - } - return 0 -} - func (m *FieldOptions) GetTimeQuantum() string { if m != nil { return m.TimeQuantum @@ -178,45 +146,42 @@ func (m *FieldOptions) GetNoStandardView() bool { return false } -type ImportResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ImportResponse) Reset() { *m = ImportResponse{} } -func (m *ImportResponse) String() string { return proto.CompactTextString(m) } -func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{2} -} -func (m *ImportResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil +func (m *FieldOptions) GetBase() int64 { + if m != nil { + return m.Base } -} -func (dst *ImportResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportResponse.Merge(dst, src) -} -func (m *ImportResponse) XXX_Size() int { - return m.Size() -} -func (m *ImportResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ImportResponse.DiscardUnknown(m) + return 0 } -var xxx_messageInfo_ImportResponse proto.InternalMessageInfo +func (m *FieldOptions) GetBitDepth() uint64 { + if m != nil { + return m.BitDepth + } + return 0 +} + +func (m *FieldOptions) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *FieldOptions) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + +type ImportResponse struct { + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` +} + +func (m *ImportResponse) Reset() { *m = ImportResponse{} } +func (m *ImportResponse) String() string { return proto.CompactTextString(m) } +func (*ImportResponse) ProtoMessage() {} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} } func (m *ImportResponse) GetErr() string { if m != nil { @@ -226,48 +191,17 @@ func (m *ImportResponse) GetErr() string { } type BlockDataRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` - Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` - Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` + Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } -func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } -func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } -func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{3} -} -func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BlockDataRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BlockDataRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BlockDataRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataRequest.Merge(dst, src) -} -func (m *BlockDataRequest) XXX_Size() int { - return m.Size() -} -func (m *BlockDataRequest) XXX_DiscardUnknown() { - xxx_messageInfo_BlockDataRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_BlockDataRequest proto.InternalMessageInfo +func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } +func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } +func (*BlockDataRequest) ProtoMessage() {} +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } func (m *BlockDataRequest) GetIndex() string { if m != nil { @@ -305,45 +239,14 @@ func (m *BlockDataRequest) GetBlock() uint64 { } type BlockDataResponse struct { - RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` } -func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } -func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } -func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{4} -} -func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BlockDataResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BlockDataResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BlockDataResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataResponse.Merge(dst, src) -} -func (m *BlockDataResponse) XXX_Size() int { - return m.Size() -} -func (m *BlockDataResponse) XXX_DiscardUnknown() { - xxx_messageInfo_BlockDataResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_BlockDataResponse proto.InternalMessageInfo +func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } +func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } +func (*BlockDataResponse) ProtoMessage() {} +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } func (m *BlockDataResponse) GetRowIDs() []uint64 { if m != nil { @@ -360,44 +263,13 @@ func (m *BlockDataResponse) GetColumnIDs() []uint64 { } type Cache struct { - IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } -func (m *Cache) Reset() { *m = Cache{} } -func (m *Cache) String() string { return proto.CompactTextString(m) } -func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{5} -} -func (m *Cache) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Cache) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Cache.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Cache) XXX_Merge(src proto.Message) { - xxx_messageInfo_Cache.Merge(dst, src) -} -func (m *Cache) XXX_Size() int { - return m.Size() -} -func (m *Cache) XXX_DiscardUnknown() { - xxx_messageInfo_Cache.DiscardUnknown(m) -} - -var xxx_messageInfo_Cache proto.InternalMessageInfo +func (m *Cache) Reset() { *m = Cache{} } +func (m *Cache) String() string { return proto.CompactTextString(m) } +func (*Cache) ProtoMessage() {} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{5} } func (m *Cache) GetIDs() []uint64 { if m != nil { @@ -407,44 +279,13 @@ func (m *Cache) GetIDs() []uint64 { } type MaxShards struct { - Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxShards) Reset() { *m = MaxShards{} } -func (m *MaxShards) String() string { return proto.CompactTextString(m) } -func (*MaxShards) ProtoMessage() {} -func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{6} -} -func (m *MaxShards) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *MaxShards) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_MaxShards.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *MaxShards) XXX_Merge(src proto.Message) { - xxx_messageInfo_MaxShards.Merge(dst, src) -} -func (m *MaxShards) XXX_Size() int { - return m.Size() -} -func (m *MaxShards) XXX_DiscardUnknown() { - xxx_messageInfo_MaxShards.DiscardUnknown(m) -} - -var xxx_messageInfo_MaxShards proto.InternalMessageInfo +func (m *MaxShards) Reset() { *m = MaxShards{} } +func (m *MaxShards) String() string { return proto.CompactTextString(m) } +func (*MaxShards) ProtoMessage() {} +func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } func (m *MaxShards) GetStandard() map[string]uint64 { if m != nil { @@ -454,46 +295,15 @@ func (m *MaxShards) GetStandard() map[string]uint64 { } type CreateShardMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } -func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } -func (*CreateShardMessage) ProtoMessage() {} -func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{7} -} -func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateShardMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateShardMessage.Merge(dst, src) -} -func (m *CreateShardMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateShardMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateShardMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateShardMessage proto.InternalMessageInfo +func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } +func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } +func (*CreateShardMessage) ProtoMessage() {} +func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } func (m *CreateShardMessage) GetIndex() string { if m != nil { @@ -517,44 +327,13 @@ func (m *CreateShardMessage) GetShard() uint64 { } type DeleteIndexMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } -func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } -func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteIndexMessage) ProtoMessage() {} -func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{8} -} -func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteIndexMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteIndexMessage.Merge(dst, src) -} -func (m *DeleteIndexMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteIndexMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteIndexMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteIndexMessage proto.InternalMessageInfo +func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } +func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteIndexMessage) ProtoMessage() {} +func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } func (m *DeleteIndexMessage) GetIndex() string { if m != nil { @@ -564,45 +343,14 @@ func (m *DeleteIndexMessage) GetIndex() string { } type CreateIndexMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } -func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } -func (*CreateIndexMessage) ProtoMessage() {} -func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{9} -} -func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateIndexMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateIndexMessage.Merge(dst, src) -} -func (m *CreateIndexMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateIndexMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateIndexMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateIndexMessage proto.InternalMessageInfo +func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } +func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } +func (*CreateIndexMessage) ProtoMessage() {} +func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } func (m *CreateIndexMessage) GetIndex() string { if m != nil { @@ -619,46 +367,15 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { } type CreateFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } -func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFieldMessage) ProtoMessage() {} -func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{10} -} -func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateFieldMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateFieldMessage.Merge(dst, src) -} -func (m *CreateFieldMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateFieldMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateFieldMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateFieldMessage proto.InternalMessageInfo +func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } +func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFieldMessage) ProtoMessage() {} +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } func (m *CreateFieldMessage) GetIndex() string { if m != nil { @@ -682,45 +399,14 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { } type DeleteFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` } -func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } -func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFieldMessage) ProtoMessage() {} -func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{11} -} -func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteFieldMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteFieldMessage.Merge(dst, src) -} -func (m *DeleteFieldMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteFieldMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteFieldMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteFieldMessage proto.InternalMessageInfo +func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } +func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFieldMessage) ProtoMessage() {} +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } func (m *DeleteFieldMessage) GetIndex() string { if m != nil { @@ -737,46 +423,17 @@ func (m *DeleteFieldMessage) GetField() string { } type DeleteAvailableShardMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - ShardID uint64 `protobuf:"varint,3,opt,name=ShardID,proto3" json:"ShardID,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + ShardID uint64 `protobuf:"varint,3,opt,name=ShardID,proto3" json:"ShardID,omitempty"` } func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShardMessage{} } func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{12} + return fileDescriptorPrivate, []int{12} } -func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteAvailableShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteAvailableShardMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteAvailableShardMessage.Merge(dst, src) -} -func (m *DeleteAvailableShardMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteAvailableShardMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteAvailableShardMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteAvailableShardMessage proto.InternalMessageInfo func (m *DeleteAvailableShardMessage) GetIndex() string { if m != nil { @@ -800,46 +457,15 @@ func (m *DeleteAvailableShardMessage) GetShardID() uint64 { } type Field struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } -func (m *Field) Reset() { *m = Field{} } -func (m *Field) String() string { return proto.CompactTextString(m) } -func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{13} -} -func (m *Field) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Field.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) -} -func (m *Field) XXX_Size() int { - return m.Size() -} -func (m *Field) XXX_DiscardUnknown() { - xxx_messageInfo_Field.DiscardUnknown(m) -} - -var xxx_messageInfo_Field proto.InternalMessageInfo +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } func (m *Field) GetName() string { if m != nil { @@ -863,44 +489,13 @@ func (m *Field) GetViews() []string { } type Schema struct { - Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` } -func (m *Schema) Reset() { *m = Schema{} } -func (m *Schema) String() string { return proto.CompactTextString(m) } -func (*Schema) ProtoMessage() {} -func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{14} -} -func (m *Schema) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Schema.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) -} -func (m *Schema) XXX_Size() int { - return m.Size() -} -func (m *Schema) XXX_DiscardUnknown() { - xxx_messageInfo_Schema.DiscardUnknown(m) -} - -var xxx_messageInfo_Schema proto.InternalMessageInfo +func (m *Schema) Reset() { *m = Schema{} } +func (m *Schema) String() string { return proto.CompactTextString(m) } +func (*Schema) ProtoMessage() {} +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Schema) GetIndexes() []*Index { if m != nil { @@ -910,45 +505,14 @@ func (m *Schema) GetIndexes() []*Index { } type Index struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` } -func (m *Index) Reset() { *m = Index{} } -func (m *Index) String() string { return proto.CompactTextString(m) } -func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{15} -} -func (m *Index) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Index.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Index) XXX_Merge(src proto.Message) { - xxx_messageInfo_Index.Merge(dst, src) -} -func (m *Index) XXX_Size() int { - return m.Size() -} -func (m *Index) XXX_DiscardUnknown() { - xxx_messageInfo_Index.DiscardUnknown(m) -} - -var xxx_messageInfo_Index proto.InternalMessageInfo +func (m *Index) Reset() { *m = Index{} } +func (m *Index) String() string { return proto.CompactTextString(m) } +func (*Index) ProtoMessage() {} +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *Index) GetName() string { if m != nil { @@ -965,46 +529,15 @@ func (m *Index) GetFields() []*Field { } type URI struct { - Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` - Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` - Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *URI) Reset() { *m = URI{} } -func (m *URI) String() string { return proto.CompactTextString(m) } -func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{16} -} -func (m *URI) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *URI) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_URI.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *URI) XXX_Merge(src proto.Message) { - xxx_messageInfo_URI.Merge(dst, src) -} -func (m *URI) XXX_Size() int { - return m.Size() -} -func (m *URI) XXX_DiscardUnknown() { - xxx_messageInfo_URI.DiscardUnknown(m) -} - -var xxx_messageInfo_URI proto.InternalMessageInfo +func (m *URI) Reset() { *m = URI{} } +func (m *URI) String() string { return proto.CompactTextString(m) } +func (*URI) ProtoMessage() {} +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *URI) GetScheme() string { if m != nil { @@ -1028,47 +561,16 @@ func (m *URI) GetPort() uint32 { } type Node struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` - IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` - State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` + State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` } -func (m *Node) Reset() { *m = Node{} } -func (m *Node) String() string { return proto.CompactTextString(m) } -func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{17} -} -func (m *Node) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Node) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Node.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Node) XXX_Merge(src proto.Message) { - xxx_messageInfo_Node.Merge(dst, src) -} -func (m *Node) XXX_Size() int { - return m.Size() -} -func (m *Node) XXX_DiscardUnknown() { - xxx_messageInfo_Node.DiscardUnknown(m) -} - -var xxx_messageInfo_Node proto.InternalMessageInfo +func (m *Node) Reset() { *m = Node{} } +func (m *Node) String() string { return proto.CompactTextString(m) } +func (*Node) ProtoMessage() {} +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *Node) GetID() string { if m != nil { @@ -1099,45 +601,14 @@ func (m *Node) GetState() string { } type NodeStateMessage struct { - NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` } -func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } -func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } -func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{18} -} -func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeStateMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeStateMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeStateMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStateMessage.Merge(dst, src) -} -func (m *NodeStateMessage) XXX_Size() int { - return m.Size() -} -func (m *NodeStateMessage) XXX_DiscardUnknown() { - xxx_messageInfo_NodeStateMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeStateMessage proto.InternalMessageInfo +func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } +func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } +func (*NodeStateMessage) ProtoMessage() {} +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeStateMessage) GetNodeID() string { if m != nil { @@ -1154,45 +625,14 @@ func (m *NodeStateMessage) GetState() string { } type NodeEventMessage struct { - Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` } -func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } -func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } -func (*NodeEventMessage) ProtoMessage() {} -func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{19} -} -func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeEventMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeEventMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeEventMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeEventMessage.Merge(dst, src) -} -func (m *NodeEventMessage) XXX_Size() int { - return m.Size() -} -func (m *NodeEventMessage) XXX_DiscardUnknown() { - xxx_messageInfo_NodeEventMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeEventMessage proto.InternalMessageInfo +func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } +func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } +func (*NodeEventMessage) ProtoMessage() {} +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeEventMessage) GetEvent() uint32 { if m != nil { @@ -1209,46 +649,15 @@ func (m *NodeEventMessage) GetNode() *Node { } type NodeStatus struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` - Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` + Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` } -func (m *NodeStatus) Reset() { *m = NodeStatus{} } -func (m *NodeStatus) String() string { return proto.CompactTextString(m) } -func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{20} -} -func (m *NodeStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NodeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NodeStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *NodeStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStatus.Merge(dst, src) -} -func (m *NodeStatus) XXX_Size() int { - return m.Size() -} -func (m *NodeStatus) XXX_DiscardUnknown() { - xxx_messageInfo_NodeStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_NodeStatus proto.InternalMessageInfo +func (m *NodeStatus) Reset() { *m = NodeStatus{} } +func (m *NodeStatus) String() string { return proto.CompactTextString(m) } +func (*NodeStatus) ProtoMessage() {} +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -1272,45 +681,14 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { } type IndexStatus struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` } -func (m *IndexStatus) Reset() { *m = IndexStatus{} } -func (m *IndexStatus) String() string { return proto.CompactTextString(m) } -func (*IndexStatus) ProtoMessage() {} -func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{21} -} -func (m *IndexStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *IndexStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_IndexStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *IndexStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexStatus.Merge(dst, src) -} -func (m *IndexStatus) XXX_Size() int { - return m.Size() -} -func (m *IndexStatus) XXX_DiscardUnknown() { - xxx_messageInfo_IndexStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_IndexStatus proto.InternalMessageInfo +func (m *IndexStatus) Reset() { *m = IndexStatus{} } +func (m *IndexStatus) String() string { return proto.CompactTextString(m) } +func (*IndexStatus) ProtoMessage() {} +func (*IndexStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } func (m *IndexStatus) GetName() string { if m != nil { @@ -1327,45 +705,14 @@ func (m *IndexStatus) GetFields() []*FieldStatus { } type FieldStatus struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` } -func (m *FieldStatus) Reset() { *m = FieldStatus{} } -func (m *FieldStatus) String() string { return proto.CompactTextString(m) } -func (*FieldStatus) ProtoMessage() {} -func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{22} -} -func (m *FieldStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldStatus.Merge(dst, src) -} -func (m *FieldStatus) XXX_Size() int { - return m.Size() -} -func (m *FieldStatus) XXX_DiscardUnknown() { - xxx_messageInfo_FieldStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldStatus proto.InternalMessageInfo +func (m *FieldStatus) Reset() { *m = FieldStatus{} } +func (m *FieldStatus) String() string { return proto.CompactTextString(m) } +func (*FieldStatus) ProtoMessage() {} +func (*FieldStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *FieldStatus) GetName() string { if m != nil { @@ -1382,46 +729,15 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { } type ClusterStatus struct { - ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` } -func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } -func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } -func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{23} -} -func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ClusterStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ClusterStatus.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ClusterStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClusterStatus.Merge(dst, src) -} -func (m *ClusterStatus) XXX_Size() int { - return m.Size() -} -func (m *ClusterStatus) XXX_DiscardUnknown() { - xxx_messageInfo_ClusterStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_ClusterStatus proto.InternalMessageInfo +func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } +func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } +func (*ClusterStatus) ProtoMessage() {} +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -1445,47 +761,16 @@ func (m *ClusterStatus) GetNodes() []*Node { } type BSIGroup struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` - Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` - Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` + Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *BSIGroup) Reset() { *m = BSIGroup{} } -func (m *BSIGroup) String() string { return proto.CompactTextString(m) } -func (*BSIGroup) ProtoMessage() {} -func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{24} -} -func (m *BSIGroup) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BSIGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BSIGroup.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *BSIGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_BSIGroup.Merge(dst, src) -} -func (m *BSIGroup) XXX_Size() int { - return m.Size() -} -func (m *BSIGroup) XXX_DiscardUnknown() { - xxx_messageInfo_BSIGroup.DiscardUnknown(m) -} - -var xxx_messageInfo_BSIGroup proto.InternalMessageInfo +func (m *BSIGroup) Reset() { *m = BSIGroup{} } +func (m *BSIGroup) String() string { return proto.CompactTextString(m) } +func (*BSIGroup) ProtoMessage() {} +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *BSIGroup) GetName() string { if m != nil { @@ -1516,46 +801,15 @@ func (m *BSIGroup) GetMax() int64 { } type CreateViewMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } -func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } -func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } -func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{25} -} -func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *CreateViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_CreateViewMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *CreateViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateViewMessage.Merge(dst, src) -} -func (m *CreateViewMessage) XXX_Size() int { - return m.Size() -} -func (m *CreateViewMessage) XXX_DiscardUnknown() { - xxx_messageInfo_CreateViewMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_CreateViewMessage proto.InternalMessageInfo +func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } +func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } +func (*CreateViewMessage) ProtoMessage() {} +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -1579,46 +833,15 @@ func (m *CreateViewMessage) GetView() string { } type DeleteViewMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } -func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } -func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{26} -} -func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeleteViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeleteViewMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *DeleteViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteViewMessage.Merge(dst, src) -} -func (m *DeleteViewMessage) XXX_Size() int { - return m.Size() -} -func (m *DeleteViewMessage) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteViewMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteViewMessage proto.InternalMessageInfo +func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } +func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteViewMessage) ProtoMessage() {} +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -1642,49 +865,18 @@ func (m *DeleteViewMessage) GetView() string { } type ResizeInstruction struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` - Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` - ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` } -func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } -func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } -func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{27} -} -func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeInstruction) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeInstruction.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeInstruction) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstruction.Merge(dst, src) -} -func (m *ResizeInstruction) XXX_Size() int { - return m.Size() -} -func (m *ResizeInstruction) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeInstruction.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeInstruction proto.InternalMessageInfo +func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } +func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } +func (*ResizeInstruction) ProtoMessage() {} +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -1729,48 +921,17 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { } type ResizeSource struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` - View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *ResizeSource) Reset() { *m = ResizeSource{} } -func (m *ResizeSource) String() string { return proto.CompactTextString(m) } -func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{28} -} -func (m *ResizeSource) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeSource.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeSource.Merge(dst, src) -} -func (m *ResizeSource) XXX_Size() int { - return m.Size() -} -func (m *ResizeSource) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeSource.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeSource proto.InternalMessageInfo +func (m *ResizeSource) Reset() { *m = ResizeSource{} } +func (m *ResizeSource) String() string { return proto.CompactTextString(m) } +func (*ResizeSource) ProtoMessage() {} +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -1808,46 +969,17 @@ func (m *ResizeSource) GetShard() uint64 { } type ResizeInstructionComplete struct { - JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` } func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{29} + return fileDescriptorPrivate, []int{29} } -func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ResizeInstructionComplete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ResizeInstructionComplete.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ResizeInstructionComplete) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstructionComplete.Merge(dst, src) -} -func (m *ResizeInstructionComplete) XXX_Size() int { - return m.Size() -} -func (m *ResizeInstructionComplete) XXX_DiscardUnknown() { - xxx_messageInfo_ResizeInstructionComplete.DiscardUnknown(m) -} - -var xxx_messageInfo_ResizeInstructionComplete proto.InternalMessageInfo func (m *ResizeInstructionComplete) GetJobID() int64 { if m != nil { @@ -1871,44 +1003,13 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` } -func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } -func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } -func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{30} -} -func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SetCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SetCoordinatorMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *SetCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_SetCoordinatorMessage.Merge(dst, src) -} -func (m *SetCoordinatorMessage) XXX_Size() int { - return m.Size() -} -func (m *SetCoordinatorMessage) XXX_DiscardUnknown() { - xxx_messageInfo_SetCoordinatorMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_SetCoordinatorMessage proto.InternalMessageInfo +func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } +func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*SetCoordinatorMessage) ProtoMessage() {} +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -1918,44 +1019,13 @@ func (m *SetCoordinatorMessage) GetNew() *Node { } type UpdateCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` } -func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } -func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } -func (*UpdateCoordinatorMessage) ProtoMessage() {} -func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{31} -} -func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *UpdateCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_UpdateCoordinatorMessage.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_UpdateCoordinatorMessage.Merge(dst, src) -} -func (m *UpdateCoordinatorMessage) XXX_Size() int { - return m.Size() -} -func (m *UpdateCoordinatorMessage) XXX_DiscardUnknown() { - xxx_messageInfo_UpdateCoordinatorMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_UpdateCoordinatorMessage proto.InternalMessageInfo +func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } +func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*UpdateCoordinatorMessage) ProtoMessage() {} +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } func (m *UpdateCoordinatorMessage) GetNew() *Node { if m != nil { @@ -1965,45 +1035,14 @@ func (m *UpdateCoordinatorMessage) GetNew() *Node { } type Topology struct { - ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` } -func (m *Topology) Reset() { *m = Topology{} } -func (m *Topology) String() string { return proto.CompactTextString(m) } -func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{32} -} -func (m *Topology) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Topology.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Topology) XXX_Merge(src proto.Message) { - xxx_messageInfo_Topology.Merge(dst, src) -} -func (m *Topology) XXX_Size() int { - return m.Size() -} -func (m *Topology) XXX_DiscardUnknown() { - xxx_messageInfo_Topology.DiscardUnknown(m) -} - -var xxx_messageInfo_Topology proto.InternalMessageInfo +func (m *Topology) Reset() { *m = Topology{} } +func (m *Topology) String() string { return proto.CompactTextString(m) } +func (*Topology) ProtoMessage() {} +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } func (m *Topology) GetClusterID() string { if m != nil { @@ -2020,43 +1059,12 @@ func (m *Topology) GetNodeIDs() []string { } type RecalculateCaches struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } -func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } -func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_8095a89af06a70de, []int{33} -} -func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RecalculateCaches) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RecalculateCaches.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *RecalculateCaches) XXX_Merge(src proto.Message) { - xxx_messageInfo_RecalculateCaches.Merge(dst, src) -} -func (m *RecalculateCaches) XXX_Size() int { - return m.Size() -} -func (m *RecalculateCaches) XXX_DiscardUnknown() { - xxx_messageInfo_RecalculateCaches.DiscardUnknown(m) -} - -var xxx_messageInfo_RecalculateCaches proto.InternalMessageInfo +func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } +func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } +func (*RecalculateCaches) ProtoMessage() {} +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{33} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") @@ -2066,7 +1074,6 @@ func init() { proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxShards)(nil), "internal.MaxShards") - proto.RegisterMapType((map[string]uint64)(nil), "internal.MaxShards.StandardEntry") proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") @@ -2130,9 +1137,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2204,8 +1208,15 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Base != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) + } + if m.BitDepth != 0 { + dAtA[i] = 0x70 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) } return i, nil } @@ -2231,9 +1242,6 @@ func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) i += copy(dAtA[i:], m.Err) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2280,9 +1288,6 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2335,9 +1340,6 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j3)) i += copy(dAtA[i:], dAtA4[:j3]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2373,9 +1375,6 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j5)) i += copy(dAtA[i:], dAtA6[:j5]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2410,9 +1409,6 @@ func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(v)) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2448,9 +1444,6 @@ func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2475,9 +1468,6 @@ func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2512,9 +1502,6 @@ func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { } i += n7 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2555,9 +1542,6 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { } i += n8 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2588,9 +1572,6 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2626,9 +1607,6 @@ func (m *DeleteAvailableShardMessage) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2678,9 +1656,6 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2711,9 +1686,6 @@ func (m *Schema) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2750,9 +1722,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2788,9 +1757,6 @@ func (m *URI) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2841,9 +1807,6 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2874,9 +1837,6 @@ func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2910,9 +1870,6 @@ func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { } i += n11 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2963,9 +1920,6 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3002,9 +1956,6 @@ func (m *IndexStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3046,9 +1997,6 @@ func (m *FieldStatus) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j14)) i += copy(dAtA[i:], dAtA15[:j14]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3091,9 +2039,6 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3134,9 +2079,6 @@ func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3173,9 +2115,6 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3212,9 +2151,6 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3290,9 +2226,6 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { } i += n19 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3344,9 +2277,6 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3386,9 +2316,6 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) i += copy(dAtA[i:], m.Error) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3417,9 +2344,6 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { } i += n22 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3448,9 +2372,6 @@ func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { } i += n23 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3490,9 +2411,6 @@ func (m *Topology) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3511,9 +2429,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -3527,9 +2442,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *IndexMeta) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Keys { @@ -3538,16 +2450,10 @@ func (m *IndexMeta) Size() (n int) { if m.TrackExistence { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldOptions) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.CacheType) @@ -3577,32 +2483,26 @@ func (m *FieldOptions) Size() (n int) { if m.NoStandardView { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) + if m.Base != 0 { + n += 1 + sovPrivate(uint64(m.Base)) + } + if m.BitDepth != 0 { + n += 1 + sovPrivate(uint64(m.BitDepth)) } return n } func (m *ImportResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Err) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BlockDataRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3623,16 +2523,10 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BlockDataResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.RowIDs) > 0 { @@ -3649,16 +2543,10 @@ func (m *BlockDataResponse) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Cache) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.IDs) > 0 { @@ -3668,16 +2556,10 @@ func (m *Cache) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *MaxShards) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Standard) > 0 { @@ -3688,16 +2570,10 @@ func (m *MaxShards) Size() (n int) { n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateShardMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3711,32 +2587,20 @@ func (m *CreateShardMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteIndexMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateIndexMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3747,16 +2611,10 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateFieldMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3771,16 +2629,10 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteFieldMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3791,16 +2643,10 @@ func (m *DeleteFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteAvailableShardMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -3814,16 +2660,10 @@ func (m *DeleteAvailableShardMessage) Size() (n int) { if m.ShardID != 0 { n += 1 + sovPrivate(uint64(m.ShardID)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Field) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -3840,16 +2680,10 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Schema) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Indexes) > 0 { @@ -3858,16 +2692,10 @@ func (m *Schema) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Index) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -3880,16 +2708,10 @@ func (m *Index) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *URI) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Scheme) @@ -3903,16 +2725,10 @@ func (m *URI) Size() (n int) { if m.Port != 0 { n += 1 + sovPrivate(uint64(m.Port)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Node) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ID) @@ -3930,16 +2746,10 @@ func (m *Node) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeStateMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.NodeID) @@ -3950,16 +2760,10 @@ func (m *NodeStateMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeEventMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Event != 0 { @@ -3969,16 +2773,10 @@ func (m *NodeEventMessage) Size() (n int) { l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *NodeStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Node != nil { @@ -3995,16 +2793,10 @@ func (m *NodeStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *IndexStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4017,16 +2809,10 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4040,16 +2826,10 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ClusterStatus) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ClusterID) @@ -4066,16 +2846,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *BSIGroup) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -4092,16 +2866,10 @@ func (m *BSIGroup) Size() (n int) { if m.Max != 0 { n += 1 + sovPrivate(uint64(m.Max)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *CreateViewMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -4116,16 +2884,10 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *DeleteViewMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -4140,16 +2902,10 @@ func (m *DeleteViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeInstruction) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.JobID != 0 { @@ -4177,16 +2933,10 @@ func (m *ResizeInstruction) Size() (n int) { l = m.NodeStatus.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeSource) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Node != nil { @@ -4208,16 +2958,10 @@ func (m *ResizeSource) Size() (n int) { if m.Shard != 0 { n += 1 + sovPrivate(uint64(m.Shard)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ResizeInstructionComplete) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.JobID != 0 { @@ -4231,48 +2975,30 @@ func (m *ResizeInstructionComplete) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *SetCoordinatorMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.New != nil { l = m.New.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *UpdateCoordinatorMessage) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.New != nil { l = m.New.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Topology) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.ClusterID) @@ -4285,21 +3011,12 @@ func (m *Topology) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *RecalculateCaches) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -4397,7 +3114,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4620,6 +3336,44 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } } m.NoStandardView = bool(v != 0) + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Base", wireType) + } + m.Base = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Base |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BitDepth", wireType) + } + m.BitDepth = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BitDepth |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4632,7 +3386,6 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4712,7 +3465,6 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4888,7 +3640,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4968,17 +3719,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5041,17 +3781,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5085,7 +3814,6 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5165,17 +3893,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.IDs) == 0 { - m.IDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5209,7 +3926,6 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5367,7 +4083,6 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5495,7 +4210,6 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5575,7 +4289,6 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5688,7 +4401,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5830,7 +4542,6 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5939,7 +4650,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6067,7 +4777,6 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6209,7 +4918,6 @@ func (m *Field) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6291,7 +4999,6 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6402,7 +5109,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6530,7 +5236,6 @@ func (m *URI) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6692,7 +5397,6 @@ func (m *Node) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6801,7 +5505,6 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -6904,7 +5607,6 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7052,7 +5754,6 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7163,7 +5864,6 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7272,17 +5972,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.AvailableShards) == 0 { - m.AvailableShards = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -7316,7 +6005,6 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7456,7 +6144,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7603,7 +6290,6 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7741,7 +6427,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -7879,7 +6564,6 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8112,7 +6796,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8302,7 +6985,6 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8434,7 +7116,6 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8518,7 +7199,6 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8602,7 +7282,6 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8711,7 +7390,6 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8762,7 +7440,6 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -8877,80 +7554,82 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_8095a89af06a70de) } +func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } -var fileDescriptor_private_8095a89af06a70de = []byte{ - // 1139 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0xc5, - 0x1b, 0xff, 0xef, 0x21, 0x8e, 0xfd, 0x39, 0x4e, 0x93, 0x6d, 0x9b, 0xff, 0x16, 0x50, 0x08, 0xa3, - 0x8a, 0x86, 0x4a, 0x84, 0xaa, 0xe5, 0x82, 0x53, 0xa5, 0x92, 0x38, 0x94, 0xa5, 0x24, 0x94, 0x71, - 0x92, 0x3b, 0x2e, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0x63, 0x76, 0x67, 0x93, 0xb8, 0x17, 0xdc, - 0x82, 0xc4, 0x0b, 0xf0, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x42, 0x15, 0x5e, 0x04, 0xcd, 0x37, 0x33, - 0xbb, 0x6b, 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xbe, 0xdf, 0x77, 0x3e, 0xae, 0x0d, 0x9d, 0x51, 0x16, - 0x9f, 0x30, 0xc9, 0x37, 0x46, 0x99, 0x90, 0x22, 0x68, 0xc6, 0xa9, 0xe4, 0x59, 0xca, 0x12, 0xf2, - 0x1c, 0x5a, 0x51, 0x3a, 0xe0, 0x67, 0x3b, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x05, 0x1f, 0xe7, 0xa1, - 0xb7, 0xe6, 0xac, 0x37, 0x29, 0xbe, 0x83, 0xf7, 0x61, 0x71, 0x2f, 0x63, 0xfd, 0xe3, 0xed, 0xb3, - 0x38, 0x97, 0x3c, 0xed, 0xf3, 0xd0, 0x47, 0xee, 0x14, 0x4a, 0xde, 0x38, 0xb0, 0xf0, 0x55, 0xcc, - 0x93, 0xc1, 0x77, 0x23, 0x19, 0x8b, 0x34, 0x0f, 0xde, 0x81, 0xd6, 0x16, 0xeb, 0x1f, 0xf1, 0xbd, - 0xf1, 0x88, 0xa3, 0xc5, 0x16, 0xad, 0x80, 0x92, 0xdb, 0x8b, 0x5f, 0x6b, 0x8b, 0x1d, 0x5a, 0x01, - 0xc1, 0x1a, 0xb4, 0xf7, 0xe2, 0x21, 0xff, 0xbe, 0x60, 0xa9, 0x2c, 0x86, 0xe1, 0x1c, 0x6a, 0xd7, - 0x21, 0x15, 0x2a, 0x1a, 0x6e, 0x22, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0x3b, 0x71, 0x1a, 0xb6, 0xd6, - 0x9c, 0x75, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x2c, 0x04, 0x83, 0xb0, 0xb3, 0x32, 0xc5, 0xf6, 0x64, - 0x8a, 0xbb, 0xa2, 0x27, 0x59, 0x3a, 0x60, 0xd9, 0xe0, 0x20, 0xe6, 0xa7, 0xe1, 0x82, 0x4e, 0x71, - 0x12, 0x25, 0x04, 0x16, 0xa3, 0xe1, 0x48, 0x64, 0x92, 0xf2, 0x7c, 0x24, 0xd2, 0x1c, 0x3d, 0x6e, - 0x67, 0x59, 0xe8, 0x60, 0x10, 0xea, 0x49, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0xf4, 0x8f, 0xbb, 0x4c, - 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0x3b, 0x30, 0x87, 0x35, 0x36, 0x72, 0x9a, 0x50, 0x28, - 0xd6, 0x2b, 0x74, 0x35, 0x8a, 0x84, 0x42, 0x51, 0x1f, 0x2b, 0xe6, 0x53, 0x4d, 0x28, 0xb4, 0x77, - 0xc4, 0xb2, 0x01, 0x56, 0xca, 0xa7, 0x9a, 0x50, 0xb9, 0x60, 0xb4, 0xba, 0x3c, 0xf8, 0x26, 0x11, - 0x2c, 0xd7, 0xfc, 0x9b, 0x30, 0x57, 0xa0, 0x41, 0xc5, 0x69, 0xd4, 0xcd, 0x43, 0x67, 0xcd, 0x5b, - 0xf7, 0xa9, 0xa1, 0xb0, 0x09, 0x22, 0x29, 0x86, 0xa9, 0x62, 0xb9, 0xc8, 0xaa, 0x00, 0x72, 0x0f, - 0xe6, 0xb0, 0x23, 0x2a, 0xcb, 0x4a, 0x57, 0x3d, 0xc9, 0xcf, 0x0e, 0xb4, 0x76, 0xd8, 0x19, 0x86, - 0x91, 0x07, 0x4f, 0xa1, 0x69, 0xeb, 0x84, 0x42, 0xed, 0xc7, 0xef, 0x6d, 0xd8, 0x01, 0xdb, 0x28, - 0xc5, 0x36, 0xac, 0xcc, 0x76, 0x2a, 0xb3, 0x31, 0x2d, 0x55, 0xde, 0xfa, 0x1c, 0x3a, 0x13, 0x2c, - 0xe5, 0xef, 0x98, 0x8f, 0x6d, 0x55, 0x8f, 0xf9, 0x58, 0xe5, 0x7f, 0xc2, 0x92, 0x82, 0x63, 0xad, - 0x7c, 0xaa, 0x89, 0xcf, 0xdc, 0x4f, 0x1c, 0x72, 0x00, 0xc1, 0x56, 0xc6, 0x99, 0xe4, 0xe8, 0x64, - 0x87, 0xe7, 0x39, 0x7b, 0xc5, 0x2f, 0xaf, 0xb8, 0xae, 0xa2, 0x5b, 0xaf, 0x62, 0xd9, 0x07, 0xaf, - 0xd6, 0x07, 0xf2, 0x10, 0x82, 0x2e, 0x4f, 0xb8, 0xe4, 0x66, 0x3b, 0xfe, 0xc1, 0x2e, 0xe9, 0xd9, - 0x18, 0xae, 0x96, 0x0d, 0x1e, 0x80, 0xaf, 0x56, 0x0d, 0x43, 0x68, 0x3f, 0xbe, 0x5d, 0xd5, 0xa9, - 0xdc, 0x42, 0x8a, 0x02, 0x24, 0xb1, 0x46, 0x31, 0x9e, 0x2b, 0x13, 0x9b, 0x31, 0x4a, 0x0f, 0x8d, - 0x2b, 0x0f, 0x5d, 0xad, 0x54, 0xae, 0xea, 0x6b, 0x6a, 0xbc, 0x3d, 0xb3, 0xe9, 0xde, 0xd4, 0x1b, - 0xe9, 0xc3, 0xdb, 0xda, 0xc2, 0x97, 0x27, 0x2c, 0x4e, 0xd8, 0x61, 0x72, 0xcd, 0x8e, 0xcc, 0x08, - 0x3c, 0x84, 0x79, 0xd4, 0x8d, 0xba, 0x66, 0x0b, 0x2c, 0x49, 0x7e, 0x30, 0xf2, 0x6a, 0xf4, 0x77, - 0xd9, 0x90, 0x1b, 0x6b, 0xf8, 0x2e, 0xf3, 0x75, 0xaf, 0xce, 0x57, 0x39, 0x56, 0xeb, 0xa2, 0x4e, - 0x9d, 0xa7, 0x1c, 0x23, 0x41, 0x9e, 0x40, 0xa3, 0xd7, 0x3f, 0xe2, 0x43, 0x16, 0x7c, 0x00, 0xf3, - 0x18, 0x21, 0xcf, 0xcd, 0x44, 0xdf, 0x9a, 0xea, 0x14, 0xb5, 0x7c, 0xd2, 0x35, 0x99, 0xcd, 0x8c, - 0xe9, 0x01, 0x34, 0xd0, 0x7b, 0x1e, 0xfa, 0xd3, 0x66, 0x10, 0xa7, 0x86, 0x4d, 0xb6, 0xc1, 0xdb, - 0xa7, 0x91, 0xda, 0x54, 0x8c, 0xc0, 0x5a, 0x31, 0x94, 0xb2, 0xfd, 0xb5, 0xc8, 0xa5, 0xa9, 0x13, - 0xbe, 0x15, 0xf6, 0x52, 0x64, 0x12, 0x6b, 0xd4, 0xa1, 0xf8, 0x26, 0x39, 0xf8, 0xbb, 0x62, 0xc0, - 0x83, 0x45, 0x70, 0xa3, 0xae, 0xb1, 0xe1, 0x46, 0xdd, 0xe0, 0x5d, 0x34, 0x6f, 0x4a, 0xd3, 0xa9, - 0x82, 0xd8, 0xa7, 0x11, 0x45, 0xc7, 0xf7, 0xa1, 0x13, 0xe5, 0x5b, 0x42, 0x64, 0x83, 0x38, 0x65, - 0x52, 0x64, 0xe6, 0x1b, 0x30, 0x09, 0xe2, 0x06, 0x49, 0x26, 0xf5, 0xc5, 0x6e, 0x51, 0x4d, 0x90, - 0x67, 0xb0, 0xa4, 0x9c, 0x22, 0x61, 0xfb, 0xbd, 0x02, 0x0d, 0x85, 0x95, 0x41, 0x18, 0xaa, 0xb2, - 0xe0, 0xd6, 0x2d, 0x7c, 0xab, 0x2d, 0x6c, 0x9f, 0xf0, 0x54, 0xd6, 0x26, 0x06, 0x69, 0x34, 0xd0, - 0xa1, 0x9a, 0x08, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x58, 0x65, 0xa2, 0x50, 0x8a, 0x3c, 0xf2, 0xab, - 0x03, 0x60, 0x03, 0x2a, 0xf2, 0x52, 0xc5, 0xb9, 0x5c, 0x25, 0x58, 0xb7, 0x9d, 0x37, 0xdb, 0xb2, - 0x54, 0x49, 0x69, 0x9c, 0xda, 0xc9, 0xf8, 0xa8, 0x9a, 0x0c, 0xdd, 0xd2, 0xbb, 0x53, 0x93, 0xa1, - 0xbd, 0x56, 0xf3, 0xf1, 0x12, 0xda, 0x35, 0x7c, 0xe6, 0x94, 0x7c, 0x58, 0x4e, 0x89, 0x3b, 0x6d, - 0x12, 0x71, 0x63, 0xd2, 0xce, 0xca, 0x0b, 0x68, 0xd7, 0xe0, 0x99, 0x16, 0xd7, 0xe1, 0xd6, 0xe4, - 0x1e, 0xda, 0xfb, 0x3e, 0x0d, 0x93, 0x18, 0x3a, 0x5b, 0x49, 0x91, 0x4b, 0x9e, 0x19, 0x73, 0xea, - 0xa3, 0xa0, 0x81, 0xb2, 0x79, 0x15, 0x30, 0xbb, 0x7f, 0xc1, 0x7d, 0x98, 0x53, 0x65, 0xd4, 0xeb, - 0x74, 0xb1, 0xc6, 0x9a, 0x49, 0x0e, 0xa0, 0xb9, 0xd9, 0x8b, 0x9e, 0x67, 0xa2, 0x18, 0xcd, 0x0c, - 0xda, 0x7e, 0xd3, 0xdd, 0x8b, 0xdf, 0x74, 0xef, 0xc2, 0x37, 0xdd, 0x2f, 0xbf, 0xe9, 0xa4, 0x07, - 0xcb, 0xfa, 0x54, 0xaa, 0x2d, 0xbe, 0xc9, 0xc1, 0xb1, 0x1f, 0x52, 0xaf, 0xf6, 0x21, 0xed, 0xc1, - 0xb2, 0xbe, 0x67, 0xff, 0xa5, 0xd1, 0xdf, 0x5d, 0x58, 0xa6, 0x3c, 0x8f, 0x5f, 0xf3, 0x28, 0xcd, - 0x65, 0x56, 0xf4, 0xd5, 0x4d, 0x52, 0xfa, 0xdf, 0x88, 0x43, 0x53, 0x6d, 0x8f, 0x6a, 0xe2, 0x3a, - 0x93, 0x1e, 0x3c, 0x82, 0xf6, 0xf4, 0xce, 0x5e, 0x14, 0xad, 0x8b, 0x04, 0x8f, 0x60, 0xbe, 0x27, - 0x8a, 0xac, 0x5f, 0x8e, 0x6f, 0xed, 0x4e, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xe0, 0xe9, 0xd4, - 0x80, 0x84, 0x0d, 0xf4, 0xf2, 0xff, 0x4a, 0x6f, 0x82, 0x4d, 0xa7, 0xc6, 0xe9, 0xe3, 0xfa, 0x2e, - 0x86, 0xf3, 0xa8, 0x7b, 0x67, 0x32, 0x42, 0xa3, 0x58, 0x93, 0x23, 0xbf, 0x38, 0xb0, 0x50, 0x0f, - 0xe7, 0x5a, 0x4b, 0x5c, 0x76, 0xc7, 0x9d, 0xd9, 0x1d, 0x6f, 0x56, 0x77, 0xfc, 0xaa, 0x3b, 0xd5, - 0xef, 0x83, 0xb9, 0xda, 0xef, 0x03, 0x72, 0x0c, 0xf7, 0x2e, 0xb4, 0x6c, 0x4b, 0x0c, 0x47, 0x6a, - 0x36, 0xfe, 0x45, 0xeb, 0xd4, 0x79, 0xcb, 0x32, 0xd3, 0xb4, 0x16, 0xd5, 0x04, 0xf9, 0x14, 0xee, - 0xf6, 0xb8, 0xac, 0x35, 0xcc, 0x4e, 0xde, 0x1a, 0x78, 0xbb, 0xfc, 0xf4, 0x92, 0xf4, 0x15, 0x8b, - 0x7c, 0x01, 0xe1, 0xfe, 0x68, 0xc0, 0x24, 0xbf, 0x91, 0xf6, 0x26, 0x34, 0xf7, 0xc4, 0x48, 0x24, - 0xe2, 0xd5, 0xf8, 0x8a, 0x0b, 0x10, 0xc2, 0xbc, 0xbe, 0xe5, 0xfa, 0xa4, 0xb4, 0xa8, 0x25, 0xc9, - 0x6d, 0x35, 0xdc, 0x7d, 0x96, 0xf4, 0x8b, 0x44, 0x85, 0xa1, 0x7e, 0x3b, 0xe6, 0x9b, 0x4b, 0x7f, - 0x9c, 0xaf, 0x3a, 0x7f, 0x9e, 0xaf, 0x3a, 0x6f, 0xce, 0x57, 0x9d, 0xdf, 0xfe, 0x5a, 0xfd, 0xdf, - 0x61, 0x03, 0xff, 0x83, 0x3c, 0xf9, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xd8, 0x6d, 0x1f, 0x94, - 0x0c, 0x00, 0x00, +var fileDescriptorPrivate = []byte{ + // 1180 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, + 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x75, 0xea, 0x4c, 0xdb, 0xb0, 0x2d, 0x28, 0x98, 0x51, 0x45, + 0x4d, 0x25, 0x42, 0xd5, 0x72, 0xc1, 0xa9, 0x52, 0x71, 0x1c, 0xca, 0x52, 0x12, 0xca, 0x38, 0xc9, + 0x1d, 0x17, 0x13, 0x7b, 0xd4, 0xac, 0xb2, 0xde, 0x35, 0xbb, 0xb3, 0x49, 0xdc, 0x0b, 0x6e, 0x41, + 0xe2, 0x05, 0xfa, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x02, 0x0a, 0x2f, 0x82, 0xe6, 0x9f, 0xd9, 0x83, + 0x1d, 0x87, 0x44, 0x81, 0xbb, 0xf9, 0xbf, 0x7f, 0xfe, 0xf3, 0x61, 0x67, 0xa1, 0x35, 0x49, 0x82, + 0x63, 0x2e, 0xc5, 0xc6, 0x24, 0x89, 0x65, 0x4c, 0xea, 0x41, 0x24, 0x45, 0x12, 0xf1, 0x90, 0x3e, + 0x87, 0x86, 0x1f, 0x8d, 0xc4, 0xe9, 0xb6, 0x90, 0x9c, 0x10, 0x70, 0x5f, 0x88, 0x69, 0xea, 0x39, + 0x1d, 0xab, 0x5b, 0x67, 0x78, 0x26, 0x1f, 0xc0, 0xca, 0x6e, 0xc2, 0x87, 0x47, 0x5b, 0xa7, 0x41, + 0x2a, 0x45, 0x34, 0x14, 0x9e, 0x8b, 0xdc, 0x39, 0x94, 0xbe, 0xb1, 0xe1, 0xc6, 0xd7, 0x81, 0x08, + 0x47, 0xdf, 0x4f, 0x64, 0x10, 0x47, 0x29, 0x79, 0x17, 0x1a, 0x9b, 0x7c, 0x78, 0x28, 0x76, 0xa7, + 0x13, 0x81, 0x1a, 0x1b, 0xac, 0x04, 0x0a, 0xee, 0x20, 0x78, 0xad, 0x35, 0xb6, 0x58, 0x09, 0x90, + 0x0e, 0x34, 0x77, 0x83, 0xb1, 0xf8, 0x21, 0xe3, 0x91, 0xcc, 0xc6, 0xde, 0x12, 0x4a, 0x57, 0x21, + 0xe5, 0x2a, 0x2a, 0xae, 0x23, 0x0b, 0xcf, 0xe4, 0x36, 0x38, 0xdb, 0x41, 0xe4, 0x35, 0x3a, 0x56, + 0xd7, 0xe9, 0xd9, 0x9e, 0xc5, 0x14, 0x89, 0x28, 0x3f, 0xf5, 0xa0, 0x82, 0xf2, 0xd3, 0x22, 0xd4, + 0xe6, 0x6c, 0xa8, 0x3b, 0xf1, 0x40, 0xf2, 0x68, 0xc4, 0x93, 0xd1, 0x7e, 0x20, 0x4e, 0xbc, 0x1b, + 0x3a, 0xd4, 0x59, 0x54, 0xc9, 0xf6, 0x78, 0x2a, 0xbc, 0x96, 0x52, 0xc9, 0xf0, 0x4c, 0xee, 0x41, + 0xbd, 0x17, 0xc8, 0xbe, 0x98, 0xc8, 0x43, 0x6f, 0xa5, 0x63, 0x75, 0x5d, 0x56, 0xd0, 0x94, 0xc2, + 0x8a, 0x3f, 0x9e, 0xc4, 0x89, 0x64, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48, 0x1b, 0x9c, 0xad, 0x24, + 0xf1, 0x2c, 0x74, 0x5e, 0x1d, 0xe9, 0xcf, 0xd0, 0xee, 0x85, 0xf1, 0xf0, 0xa8, 0xcf, 0x25, 0x67, + 0xe2, 0xa7, 0x4c, 0xa4, 0x92, 0xdc, 0x86, 0x25, 0xac, 0x8d, 0xb9, 0xa7, 0x09, 0x85, 0x62, 0x9e, + 0x3d, 0x5b, 0xa3, 0x48, 0x28, 0x14, 0xe5, 0x31, 0xd3, 0x2e, 0xd3, 0x84, 0x42, 0x07, 0x87, 0x3c, + 0x19, 0x61, 0x86, 0x5d, 0xa6, 0x09, 0xe5, 0x3f, 0x46, 0xa7, 0xd3, 0x8a, 0x67, 0xea, 0xc3, 0x6a, + 0xc5, 0xbe, 0x71, 0x73, 0x0d, 0x6a, 0x2c, 0x3e, 0xf1, 0xfb, 0xa9, 0x67, 0x75, 0x9c, 0xae, 0xcb, + 0x0c, 0x85, 0xc5, 0x8b, 0xc3, 0x6c, 0x1c, 0x29, 0x96, 0x8d, 0xac, 0x12, 0xa0, 0x77, 0x61, 0x09, + 0x2b, 0xa9, 0xa2, 0x2c, 0x65, 0xd5, 0x91, 0xfe, 0x62, 0x41, 0x63, 0x9b, 0x9f, 0xa2, 0x1b, 0x29, + 0x79, 0x0a, 0xf5, 0x3c, 0xaf, 0x78, 0xa9, 0xf9, 0xf8, 0xfd, 0x8d, 0xbc, 0x31, 0x37, 0x8a, 0x6b, + 0x1b, 0xf9, 0x9d, 0xad, 0x48, 0x26, 0x53, 0x56, 0x88, 0xdc, 0xfb, 0x02, 0x5a, 0x33, 0x2c, 0x65, + 0xef, 0x48, 0x4c, 0xf3, 0xac, 0x1e, 0x89, 0xa9, 0x8a, 0xff, 0x98, 0x87, 0x99, 0xc0, 0x5c, 0xb9, + 0x4c, 0x13, 0x9f, 0xdb, 0x9f, 0x5a, 0x74, 0x1f, 0xc8, 0x66, 0x22, 0xb8, 0x14, 0x68, 0x64, 0x5b, + 0xa4, 0x29, 0x7f, 0x25, 0x2e, 0xce, 0xb8, 0xce, 0xa2, 0x5d, 0xcd, 0x62, 0x51, 0x07, 0xa7, 0x52, + 0x07, 0xfa, 0x10, 0x48, 0x5f, 0x84, 0x42, 0x0a, 0x33, 0x55, 0xff, 0xa2, 0x97, 0x0e, 0x72, 0x1f, + 0x2e, 0xbf, 0x4b, 0x1e, 0x80, 0xab, 0x46, 0x14, 0x5d, 0x68, 0x3e, 0xbe, 0x55, 0xe6, 0xa9, 0x98, + 0x5e, 0x86, 0x17, 0x68, 0x98, 0x2b, 0x45, 0x7f, 0x2e, 0x0d, 0x6c, 0x41, 0x2b, 0x3d, 0x34, 0xa6, + 0x1c, 0x34, 0xb5, 0x56, 0x9a, 0xaa, 0x8e, 0xb7, 0xb1, 0xf6, 0x2c, 0x0f, 0xf7, 0xba, 0xd6, 0xe8, + 0x10, 0xde, 0xd1, 0x1a, 0xbe, 0x3a, 0xe6, 0x41, 0xc8, 0x0f, 0xc2, 0x2b, 0x56, 0x64, 0x81, 0xe3, + 0x1e, 0x2c, 0xa3, 0xac, 0xdf, 0x37, 0x53, 0x90, 0x93, 0xf4, 0x47, 0x73, 0x5f, 0xb5, 0xfe, 0x0e, + 0x1f, 0x0b, 0xa3, 0x0d, 0xcf, 0x45, 0xbc, 0xf6, 0xe5, 0xf1, 0x2a, 0xc3, 0x6a, 0x5c, 0xd4, 0x8a, + 0x74, 0x94, 0x61, 0x24, 0xe8, 0x13, 0xa8, 0x0d, 0x86, 0x87, 0x62, 0xcc, 0xc9, 0x87, 0xb0, 0x8c, + 0x1e, 0x8a, 0xd4, 0x74, 0xf4, 0xcd, 0xb9, 0x4a, 0xb1, 0x9c, 0x4f, 0xfb, 0x26, 0xb2, 0x85, 0x3e, + 0x3d, 0x80, 0x1a, 0x5a, 0x4f, 0x3d, 0x77, 0x5e, 0x0d, 0xe2, 0xcc, 0xb0, 0xe9, 0x16, 0x38, 0x7b, + 0xcc, 0x57, 0x93, 0x8a, 0x1e, 0xe4, 0x5a, 0x0c, 0xa5, 0x74, 0x7f, 0x13, 0xa7, 0xd2, 0xe4, 0x09, + 0xcf, 0x0a, 0x7b, 0x19, 0x27, 0x12, 0x73, 0xd4, 0x62, 0x78, 0xa6, 0x29, 0xb8, 0x3b, 0xf1, 0x48, + 0x90, 0x15, 0xb0, 0xfd, 0xbe, 0xd1, 0x61, 0xfb, 0x7d, 0xf2, 0x1e, 0xaa, 0x37, 0xa9, 0x69, 0x95, + 0x4e, 0xec, 0x31, 0x9f, 0xa1, 0xe1, 0xfb, 0xd0, 0xf2, 0xd3, 0xcd, 0x38, 0x4e, 0x46, 0x41, 0xc4, + 0x65, 0x9c, 0x98, 0x6f, 0xc7, 0x2c, 0x88, 0x13, 0x24, 0xb9, 0xd4, 0x9b, 0xbe, 0xc1, 0x34, 0x41, + 0x9f, 0x41, 0x5b, 0x19, 0x45, 0x22, 0xaf, 0xf7, 0x1a, 0xd4, 0x14, 0x56, 0x38, 0x61, 0xa8, 0x52, + 0x83, 0x5d, 0xd5, 0xf0, 0x9d, 0xd6, 0xb0, 0x75, 0x2c, 0x22, 0x59, 0xe9, 0x18, 0xa4, 0x51, 0x41, + 0x8b, 0x69, 0x82, 0x50, 0x1d, 0xa0, 0x89, 0x64, 0xa5, 0x8c, 0x44, 0xa1, 0x0c, 0x79, 0xf4, 0x37, + 0x0b, 0x20, 0x77, 0x28, 0x4b, 0x0b, 0x11, 0xeb, 0x62, 0x11, 0xd2, 0xcd, 0x2b, 0x6f, 0xa6, 0xa5, + 0x5d, 0xde, 0xd2, 0x38, 0xcb, 0x3b, 0xe3, 0xe3, 0xb2, 0x33, 0x74, 0x49, 0xef, 0xcc, 0x75, 0x86, + 0xb6, 0x5a, 0xf6, 0xc7, 0x4b, 0x68, 0x56, 0xf0, 0x85, 0x5d, 0xf2, 0x51, 0xd1, 0x25, 0xf6, 0xbc, + 0x4a, 0xc4, 0x8d, 0xca, 0xbc, 0x57, 0x5e, 0x40, 0xb3, 0x02, 0x2f, 0xd4, 0xd8, 0x85, 0x9b, 0xb3, + 0x73, 0x98, 0xef, 0xf7, 0x79, 0x98, 0x06, 0xd0, 0xda, 0x0c, 0xb3, 0x54, 0x8a, 0xc4, 0xa8, 0x53, + 0x1f, 0x05, 0x0d, 0x14, 0xc5, 0x2b, 0x81, 0xc5, 0xf5, 0x23, 0xf7, 0x61, 0x49, 0xa5, 0x51, 0x8f, + 0xd3, 0xf9, 0x1c, 0x6b, 0x26, 0xdd, 0x87, 0x7a, 0x6f, 0xe0, 0x3f, 0x4f, 0xe2, 0x6c, 0xb2, 0xd0, + 0xe9, 0xfc, 0x2d, 0x60, 0x57, 0xde, 0x02, 0x6d, 0xfd, 0x16, 0x70, 0xf0, 0x13, 0x8d, 0xef, 0x80, + 0xb6, 0x7e, 0x07, 0xb8, 0x06, 0xe1, 0x6a, 0xff, 0xae, 0xea, 0x55, 0xa9, 0xa6, 0xf8, 0x3a, 0x0b, + 0x27, 0xff, 0x90, 0x3a, 0x95, 0x0f, 0xe9, 0x00, 0x56, 0xf5, 0x3e, 0xfb, 0x3f, 0x95, 0xfe, 0x6e, + 0xc3, 0x2a, 0x13, 0x69, 0xf0, 0x5a, 0xf8, 0x51, 0x2a, 0x93, 0x6c, 0xa8, 0x76, 0x92, 0x92, 0xff, + 0x36, 0x3e, 0x30, 0xd9, 0x76, 0x98, 0x26, 0xae, 0xd2, 0xe9, 0xe4, 0x11, 0x34, 0xe7, 0x67, 0xf6, + 0xfc, 0xd5, 0xea, 0x15, 0xf2, 0x08, 0x96, 0x07, 0x71, 0x96, 0x0c, 0x8b, 0xf6, 0xad, 0xec, 0x49, + 0xed, 0x99, 0x66, 0xb3, 0xfc, 0x1a, 0x79, 0x3a, 0xd7, 0x20, 0x5e, 0x0d, 0xad, 0xbc, 0x5d, 0xca, + 0xcd, 0xb0, 0xd9, 0x5c, 0x3b, 0x7d, 0x52, 0x9d, 0x45, 0x6f, 0x19, 0x65, 0x6f, 0xcf, 0x7a, 0x68, + 0x04, 0x2b, 0xf7, 0xe8, 0xaf, 0x16, 0xdc, 0xa8, 0xba, 0x73, 0xa5, 0x21, 0x2e, 0xaa, 0x63, 0x2f, + 0xac, 0x8e, 0xb3, 0xa8, 0x3a, 0x6e, 0x59, 0x9d, 0xf2, 0x7d, 0xb0, 0x54, 0x79, 0x1f, 0xd0, 0x23, + 0xb8, 0x7b, 0xae, 0x64, 0x9b, 0xf1, 0x78, 0xa2, 0x7a, 0xe3, 0x3f, 0x94, 0x4e, 0xad, 0xb7, 0x24, + 0x31, 0x45, 0x6b, 0x30, 0x4d, 0xd0, 0xcf, 0xe0, 0xce, 0x40, 0xc8, 0x4a, 0xc1, 0xf2, 0xce, 0xeb, + 0x80, 0xb3, 0x23, 0x4e, 0x2e, 0x08, 0x5f, 0xb1, 0xe8, 0x97, 0xe0, 0xed, 0x4d, 0x46, 0x5c, 0x8a, + 0x6b, 0x49, 0xf7, 0xa0, 0xbe, 0x1b, 0x4f, 0xe2, 0x30, 0x7e, 0x35, 0xbd, 0x64, 0x03, 0x78, 0xb0, + 0xac, 0x77, 0xb9, 0x5e, 0x29, 0x0d, 0x96, 0x93, 0xf4, 0x96, 0x6a, 0xee, 0x21, 0x0f, 0x87, 0x59, + 0xa8, 0xdc, 0x50, 0x6f, 0xc7, 0xb4, 0xd7, 0xfe, 0xe3, 0x6c, 0xdd, 0xfa, 0xf3, 0x6c, 0xdd, 0xfa, + 0xeb, 0x6c, 0xdd, 0x7a, 0xf3, 0xf7, 0xfa, 0x5b, 0x07, 0x35, 0xfc, 0x77, 0x79, 0xf2, 0x4f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x4f, 0xa0, 0xa9, 0x8b, 0xcc, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 971bb5f69..a327b835e 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -8,14 +8,16 @@ message IndexMeta { } message FieldOptions { - string Type = 8; + string Type = 8; string CacheType = 3; uint32 CacheSize = 4; - int64 Min = 9; - int64 Max = 10; string TimeQuantum = 5; - bool Keys = 11; - bool NoStandardView = 12; + int64 Min = 9; + int64 Max = 10; + bool Keys = 11; + bool NoStandardView = 12; + int64 Base = 13; + uint64 BitDepth = 14; } message ImportResponse { @@ -40,154 +42,154 @@ message Cache { } message MaxShards { - map Standard = 1; + map Standard = 1; } message CreateShardMessage { - string Index = 1; - string Field = 3; - uint64 Shard = 2; + string Index = 1; + string Field = 3; + uint64 Shard = 2; } message DeleteIndexMessage { - string Index = 1; + string Index = 1; } message CreateIndexMessage { - string Index = 1; - IndexMeta Meta = 2; + string Index = 1; + IndexMeta Meta = 2; } message CreateFieldMessage { - string Index = 1; - string Field = 2; - FieldOptions Meta = 3; + string Index = 1; + string Field = 2; + FieldOptions Meta = 3; } message DeleteFieldMessage { - string Index = 1; - string Field = 2; + string Index = 1; + string Field = 2; } message DeleteAvailableShardMessage { - string Index = 1; - string Field = 2; - uint64 ShardID = 3; + string Index = 1; + string Field = 2; + uint64 ShardID = 3; } message Field { - string Name = 1; - FieldOptions Meta = 2; - repeated string Views = 3; + string Name = 1; + FieldOptions Meta = 2; + repeated string Views = 3; } message Schema { - repeated Index Indexes = 1; + repeated Index Indexes = 1; } message Index { - string Name = 1; - repeated Field Fields = 4; + string Name = 1; + repeated Field Fields = 4; } message URI { - string Scheme = 1; - string Host = 2; - uint32 Port = 3; + string Scheme = 1; + string Host = 2; + uint32 Port = 3; } message Node { - string ID = 1; - URI URI = 2; - bool IsCoordinator = 3; - string State = 4; + string ID = 1; + URI URI = 2; + bool IsCoordinator = 3; + string State = 4; } message NodeStateMessage { - string NodeID = 1; - string State = 2; + string NodeID = 1; + string State = 2; } message NodeEventMessage { - uint32 Event = 1; - Node Node = 2; + uint32 Event = 1; + Node Node = 2; } message NodeStatus { - Node Node = 1; - Schema Schema = 3; - repeated IndexStatus Indexes = 4; + Node Node = 1; + Schema Schema = 3; + repeated IndexStatus Indexes = 4; } message IndexStatus { - string Name = 1; - repeated FieldStatus Fields = 2; + string Name = 1; + repeated FieldStatus Fields = 2; } message FieldStatus { - string Name = 1; - repeated uint64 AvailableShards = 2; + string Name = 1; + repeated uint64 AvailableShards = 2; } message ClusterStatus { - string ClusterID = 1; - string State = 2; - repeated Node Nodes = 3; + string ClusterID = 1; + string State = 2; + repeated Node Nodes = 3; } message BSIGroup { - string Name = 1; - string Type = 2; - int64 Min = 3; - int64 Max = 4; + string Name = 1; + string Type = 2; + int64 Min = 3; + int64 Max = 4; } message CreateViewMessage { - string Index = 1; - string Field = 2; - string View = 3; + string Index = 1; + string Field = 2; + string View = 3; } message DeleteViewMessage { - string Index = 1; - string Field = 2; - string View = 3; + string Index = 1; + string Field = 2; + string View = 3; } message ResizeInstruction { - int64 JobID = 1; - Node Node = 2; - Node Coordinator = 3; - repeated ResizeSource Sources = 4; - NodeStatus NodeStatus = 7; - ClusterStatus ClusterStatus = 6; + int64 JobID = 1; + Node Node = 2; + Node Coordinator = 3; + repeated ResizeSource Sources = 4; + NodeStatus NodeStatus = 7; + ClusterStatus ClusterStatus = 6; } message ResizeSource { - Node Node = 1; - string Index = 2; - string Field = 3; - string View = 4; - uint64 Shard = 5; + Node Node = 1; + string Index = 2; + string Field = 3; + string View = 4; + uint64 Shard = 5; } message ResizeInstructionComplete { - int64 JobID = 1; - Node Node = 2; - string Error = 3; + int64 JobID = 1; + Node Node = 2; + string Error = 3; } message SetCoordinatorMessage { - Node New = 1; + Node New = 1; } message UpdateCoordinatorMessage { - Node New = 1; + Node New = 1; } message Topology { - string ClusterID = 1; - repeated string NodeIDs = 2; + string ClusterID = 1; + repeated string NodeIDs = 2; } message RecalculateCaches {} diff --git a/internal/public.pb.go b/internal/public.pb.go index 5cd86a833..708d43e2f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,13 +1,39 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto +/* + Package internal is a generated protocol buffer package. + + It is generated from these files: + public.proto + + It has these top-level messages: + Row + RowIdentifiers + Pair + FieldRow + GroupCount + ValCount + ColumnAttrSet + Attr + AttrMap + QueryRequest + QueryResponse + QueryResult + ImportRequest + ImportValueRequest + TranslateKeysRequest + TranslateKeysResponse + ImportRoaringRequestView + ImportRoaringRequest +*/ package internal import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" +import binary "encoding/binary" import io "io" @@ -23,46 +49,15 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *Row) Reset() { *m = Row{} } -func (m *Row) String() string { return proto.CompactTextString(m) } -func (*Row) ProtoMessage() {} -func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{0} -} -func (m *Row) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Row.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(dst, src) -} -func (m *Row) XXX_Size() int { - return m.Size() -} -func (m *Row) XXX_DiscardUnknown() { - xxx_messageInfo_Row.DiscardUnknown(m) -} - -var xxx_messageInfo_Row proto.InternalMessageInfo +func (m *Row) Reset() { *m = Row{} } +func (m *Row) String() string { return proto.CompactTextString(m) } +func (*Row) ProtoMessage() {} +func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } func (m *Row) GetColumns() []uint64 { if m != nil { @@ -86,45 +81,14 @@ func (m *Row) GetAttrs() []*Attr { } type RowIdentifiers struct { - Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` } -func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } -func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } -func (*RowIdentifiers) ProtoMessage() {} -func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{1} -} -func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RowIdentifiers) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RowIdentifiers.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *RowIdentifiers) XXX_Merge(src proto.Message) { - xxx_messageInfo_RowIdentifiers.Merge(dst, src) -} -func (m *RowIdentifiers) XXX_Size() int { - return m.Size() -} -func (m *RowIdentifiers) XXX_DiscardUnknown() { - xxx_messageInfo_RowIdentifiers.DiscardUnknown(m) -} - -var xxx_messageInfo_RowIdentifiers proto.InternalMessageInfo +func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } +func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } +func (*RowIdentifiers) ProtoMessage() {} +func (*RowIdentifiers) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } func (m *RowIdentifiers) GetRows() []uint64 { if m != nil { @@ -141,46 +105,15 @@ func (m *RowIdentifiers) GetKeys() []string { } type Pair struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` - Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *Pair) Reset() { *m = Pair{} } -func (m *Pair) String() string { return proto.CompactTextString(m) } -func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{2} -} -func (m *Pair) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Pair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Pair.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Pair) XXX_Merge(src proto.Message) { - xxx_messageInfo_Pair.Merge(dst, src) -} -func (m *Pair) XXX_Size() int { - return m.Size() -} -func (m *Pair) XXX_DiscardUnknown() { - xxx_messageInfo_Pair.DiscardUnknown(m) -} - -var xxx_messageInfo_Pair proto.InternalMessageInfo +func (m *Pair) Reset() { *m = Pair{} } +func (m *Pair) String() string { return proto.CompactTextString(m) } +func (*Pair) ProtoMessage() {} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } func (m *Pair) GetID() uint64 { if m != nil { @@ -204,46 +137,15 @@ func (m *Pair) GetCount() uint64 { } type FieldRow struct { - Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` - RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` - RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` + RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` + RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` } -func (m *FieldRow) Reset() { *m = FieldRow{} } -func (m *FieldRow) String() string { return proto.CompactTextString(m) } -func (*FieldRow) ProtoMessage() {} -func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{3} -} -func (m *FieldRow) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FieldRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FieldRow.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *FieldRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldRow.Merge(dst, src) -} -func (m *FieldRow) XXX_Size() int { - return m.Size() -} -func (m *FieldRow) XXX_DiscardUnknown() { - xxx_messageInfo_FieldRow.DiscardUnknown(m) -} - -var xxx_messageInfo_FieldRow proto.InternalMessageInfo +func (m *FieldRow) Reset() { *m = FieldRow{} } +func (m *FieldRow) String() string { return proto.CompactTextString(m) } +func (*FieldRow) ProtoMessage() {} +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } func (m *FieldRow) GetField() string { if m != nil { @@ -267,45 +169,14 @@ func (m *FieldRow) GetRowKey() string { } type GroupCount struct { - Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *GroupCount) Reset() { *m = GroupCount{} } -func (m *GroupCount) String() string { return proto.CompactTextString(m) } -func (*GroupCount) ProtoMessage() {} -func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{4} -} -func (m *GroupCount) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *GroupCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_GroupCount.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *GroupCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_GroupCount.Merge(dst, src) -} -func (m *GroupCount) XXX_Size() int { - return m.Size() -} -func (m *GroupCount) XXX_DiscardUnknown() { - xxx_messageInfo_GroupCount.DiscardUnknown(m) -} - -var xxx_messageInfo_GroupCount proto.InternalMessageInfo +func (m *GroupCount) Reset() { *m = GroupCount{} } +func (m *GroupCount) String() string { return proto.CompactTextString(m) } +func (*GroupCount) ProtoMessage() {} +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { @@ -322,45 +193,14 @@ func (m *GroupCount) GetCount() uint64 { } type ValCount struct { - Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` - Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *ValCount) Reset() { *m = ValCount{} } -func (m *ValCount) String() string { return proto.CompactTextString(m) } -func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{5} -} -func (m *ValCount) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ValCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ValCount.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ValCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValCount.Merge(dst, src) -} -func (m *ValCount) XXX_Size() int { - return m.Size() -} -func (m *ValCount) XXX_DiscardUnknown() { - xxx_messageInfo_ValCount.DiscardUnknown(m) -} - -var xxx_messageInfo_ValCount proto.InternalMessageInfo +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -377,46 +217,15 @@ func (m *ValCount) GetCount() int64 { } type ColumnAttrSet struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` - Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } -func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } -func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{6} -} -func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ColumnAttrSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ColumnAttrSet.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ColumnAttrSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnAttrSet.Merge(dst, src) -} -func (m *ColumnAttrSet) XXX_Size() int { - return m.Size() -} -func (m *ColumnAttrSet) XXX_DiscardUnknown() { - xxx_messageInfo_ColumnAttrSet.DiscardUnknown(m) -} - -var xxx_messageInfo_ColumnAttrSet proto.InternalMessageInfo +func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } +func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } +func (*ColumnAttrSet) ProtoMessage() {} +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -440,49 +249,18 @@ func (m *ColumnAttrSet) GetAttrs() []*Attr { } type Attr struct { - Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` - StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` - IntValue int64 `protobuf:"varint,4,opt,name=IntValue,proto3" json:"IntValue,omitempty"` - BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` - FloatValue float64 `protobuf:"fixed64,6,opt,name=FloatValue,proto3" json:"FloatValue,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + IntValue int64 `protobuf:"varint,4,opt,name=IntValue,proto3" json:"IntValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` + FloatValue float64 `protobuf:"fixed64,6,opt,name=FloatValue,proto3" json:"FloatValue,omitempty"` } -func (m *Attr) Reset() { *m = Attr{} } -func (m *Attr) String() string { return proto.CompactTextString(m) } -func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{7} -} -func (m *Attr) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Attr) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Attr.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Attr) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attr.Merge(dst, src) -} -func (m *Attr) XXX_Size() int { - return m.Size() -} -func (m *Attr) XXX_DiscardUnknown() { - xxx_messageInfo_Attr.DiscardUnknown(m) -} - -var xxx_messageInfo_Attr proto.InternalMessageInfo +func (m *Attr) Reset() { *m = Attr{} } +func (m *Attr) String() string { return proto.CompactTextString(m) } +func (*Attr) ProtoMessage() {} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *Attr) GetKey() string { if m != nil { @@ -527,44 +305,13 @@ func (m *Attr) GetFloatValue() float64 { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *AttrMap) Reset() { *m = AttrMap{} } -func (m *AttrMap) String() string { return proto.CompactTextString(m) } -func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{8} -} -func (m *AttrMap) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *AttrMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_AttrMap.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *AttrMap) XXX_Merge(src proto.Message) { - xxx_messageInfo_AttrMap.Merge(dst, src) -} -func (m *AttrMap) XXX_Size() int { - return m.Size() -} -func (m *AttrMap) XXX_DiscardUnknown() { - xxx_messageInfo_AttrMap.DiscardUnknown(m) -} - -var xxx_messageInfo_AttrMap proto.InternalMessageInfo +func (m *AttrMap) Reset() { *m = AttrMap{} } +func (m *AttrMap) String() string { return proto.CompactTextString(m) } +func (*AttrMap) ProtoMessage() {} +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -574,49 +321,18 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` - ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` - ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` - ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` + ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` } -func (m *QueryRequest) Reset() { *m = QueryRequest{} } -func (m *QueryRequest) String() string { return proto.CompactTextString(m) } -func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{9} -} -func (m *QueryRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryRequest.Merge(dst, src) -} -func (m *QueryRequest) XXX_Size() int { - return m.Size() -} -func (m *QueryRequest) XXX_DiscardUnknown() { - xxx_messageInfo_QueryRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryRequest proto.InternalMessageInfo +func (m *QueryRequest) Reset() { *m = QueryRequest{} } +func (m *QueryRequest) String() string { return proto.CompactTextString(m) } +func (*QueryRequest) ProtoMessage() {} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -661,46 +377,15 @@ func (m *QueryRequest) GetExcludeColumns() bool { } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` } -func (m *QueryResponse) Reset() { *m = QueryResponse{} } -func (m *QueryResponse) String() string { return proto.CompactTextString(m) } -func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{10} -} -func (m *QueryResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResponse.Merge(dst, src) -} -func (m *QueryResponse) XXX_Size() int { - return m.Size() -} -func (m *QueryResponse) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResponse proto.InternalMessageInfo +func (m *QueryResponse) Reset() { *m = QueryResponse{} } +func (m *QueryResponse) String() string { return proto.CompactTextString(m) } +func (*QueryResponse) ProtoMessage() {} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -724,52 +409,21 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` - RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` } -func (m *QueryResult) Reset() { *m = QueryResult{} } -func (m *QueryResult) String() string { return proto.CompactTextString(m) } -func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{11} -} -func (m *QueryResult) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResult.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *QueryResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResult.Merge(dst, src) -} -func (m *QueryResult) XXX_Size() int { - return m.Size() -} -func (m *QueryResult) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResult.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResult proto.InternalMessageInfo +func (m *QueryResult) Reset() { *m = QueryResult{} } +func (m *QueryResult) String() string { return proto.CompactTextString(m) } +func (*QueryResult) ProtoMessage() {} +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -835,51 +489,20 @@ func (m *QueryResult) GetRowIdentifiers() *RowIdentifiers { } type ImportRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` - ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` + ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } -func (m *ImportRequest) Reset() { *m = ImportRequest{} } -func (m *ImportRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{12} -} -func (m *ImportRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRequest.Merge(dst, src) -} -func (m *ImportRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRequest proto.InternalMessageInfo +func (m *ImportRequest) Reset() { *m = ImportRequest{} } +func (m *ImportRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRequest) ProtoMessage() {} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -938,49 +561,18 @@ func (m *ImportRequest) GetTimestamps() []int64 { } type ImportValueRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` } -func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } -func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } -func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{13} -} -func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportValueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportValueRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportValueRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportValueRequest.Merge(dst, src) -} -func (m *ImportValueRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportValueRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportValueRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportValueRequest proto.InternalMessageInfo +func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } +func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } +func (*ImportValueRequest) ProtoMessage() {} +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -1025,46 +617,15 @@ func (m *ImportValueRequest) GetValues() []int64 { } type TranslateKeysRequest struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` } -func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } -func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } -func (*TranslateKeysRequest) ProtoMessage() {} -func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{14} -} -func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) -} -func (m *TranslateKeysRequest) XXX_Size() int { - return m.Size() -} -func (m *TranslateKeysRequest) XXX_DiscardUnknown() { - xxx_messageInfo_TranslateKeysRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_TranslateKeysRequest proto.InternalMessageInfo +func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } +func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysRequest) ProtoMessage() {} +func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{14} } func (m *TranslateKeysRequest) GetIndex() string { if m != nil { @@ -1088,44 +649,13 @@ func (m *TranslateKeysRequest) GetKeys() []string { } type TranslateKeysResponse struct { - IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` } -func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } -func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } -func (*TranslateKeysResponse) ProtoMessage() {} -func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{15} -} -func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) -} -func (m *TranslateKeysResponse) XXX_Size() int { - return m.Size() -} -func (m *TranslateKeysResponse) XXX_DiscardUnknown() { - xxx_messageInfo_TranslateKeysResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_TranslateKeysResponse proto.InternalMessageInfo +func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } +func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysResponse) ProtoMessage() {} +func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{15} } func (m *TranslateKeysResponse) GetIDs() []uint64 { if m != nil { @@ -1135,45 +665,14 @@ func (m *TranslateKeysResponse) GetIDs() []uint64 { } type ImportRoaringRequestView struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` } -func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } -func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } -func (*ImportRoaringRequestView) ProtoMessage() {} -func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{16} -} -func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) -} -func (m *ImportRoaringRequestView) XXX_Size() int { - return m.Size() -} -func (m *ImportRoaringRequestView) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRoaringRequestView.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRoaringRequestView proto.InternalMessageInfo +func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } +func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequestView) ProtoMessage() {} +func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{16} } func (m *ImportRoaringRequestView) GetName() string { if m != nil { @@ -1190,45 +689,14 @@ func (m *ImportRoaringRequestView) GetData() []byte { } type ImportRoaringRequest struct { - Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` - Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` } -func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } -func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRoaringRequest) ProtoMessage() {} -func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_f65cfea24ac19f54, []int{17} -} -func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) -} -func (m *ImportRoaringRequest) XXX_Size() int { - return m.Size() -} -func (m *ImportRoaringRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ImportRoaringRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ImportRoaringRequest proto.InternalMessageInfo +func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } +func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequest) ProtoMessage() {} +func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{17} } func (m *ImportRoaringRequest) GetClear() bool { if m != nil { @@ -1323,9 +791,6 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1376,9 +841,6 @@ func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1413,9 +875,6 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) i += copy(dAtA[i:], m.Key) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1451,9 +910,6 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) i += copy(dAtA[i:], m.RowKey) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1489,9 +945,6 @@ func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1520,9 +973,6 @@ func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1564,9 +1014,6 @@ func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) i += copy(dAtA[i:], m.Key) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1620,12 +1067,9 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1656,9 +1100,6 @@ func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1740,9 +1181,6 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1791,9 +1229,6 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -1903,9 +1338,6 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i += n11 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2023,9 +1455,6 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2111,9 +1540,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2159,9 +1585,6 @@ func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2197,9 +1620,6 @@ func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j22)) i += copy(dAtA[i:], dAtA23[:j22]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2230,9 +1650,6 @@ func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) i += copy(dAtA[i:], m.Data) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2273,9 +1690,6 @@ func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } return i, nil } @@ -2289,9 +1703,6 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *Row) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Columns) > 0 { @@ -2313,16 +1724,10 @@ func (m *Row) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *RowIdentifiers) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Rows) > 0 { @@ -2338,16 +1743,10 @@ func (m *RowIdentifiers) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Pair) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.ID != 0 { @@ -2360,16 +1759,10 @@ func (m *Pair) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *FieldRow) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Field) @@ -2383,16 +1776,10 @@ func (m *FieldRow) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *GroupCount) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Group) > 0 { @@ -2404,16 +1791,10 @@ func (m *GroupCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ValCount) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Val != 0 { @@ -2422,16 +1803,10 @@ func (m *ValCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ColumnAttrSet) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.ID != 0 { @@ -2447,16 +1822,10 @@ func (m *ColumnAttrSet) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *Attr) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Key) @@ -2479,16 +1848,10 @@ func (m *Attr) Size() (n int) { if m.FloatValue != 0 { n += 9 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *AttrMap) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.Attrs) > 0 { @@ -2497,16 +1860,10 @@ func (m *AttrMap) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Query) @@ -2532,16 +1889,10 @@ func (m *QueryRequest) Size() (n int) { if m.ExcludeColumns { n += 2 } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Err) @@ -2560,16 +1911,10 @@ func (m *QueryResponse) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *QueryResult) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Row != nil { @@ -2612,16 +1957,10 @@ func (m *QueryResult) Size() (n int) { l = m.RowIdentifiers.Size() n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2668,16 +2007,10 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportValueRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2711,16 +2044,10 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *TranslateKeysRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Index) @@ -2737,16 +2064,10 @@ func (m *TranslateKeysRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *TranslateKeysResponse) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if len(m.IDs) > 0 { @@ -2756,16 +2077,10 @@ func (m *TranslateKeysResponse) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRoaringRequestView) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l l = len(m.Name) @@ -2776,16 +2091,10 @@ func (m *ImportRoaringRequestView) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } func (m *ImportRoaringRequest) Size() (n int) { - if m == nil { - return 0 - } var l int _ = l if m.Clear { @@ -2797,9 +2106,6 @@ func (m *ImportRoaringRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -2886,17 +2192,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Columns) == 0 { - m.Columns = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -2990,7 +2285,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3070,17 +2364,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Rows) == 0 { - m.Rows = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -3143,7 +2426,6 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3261,7 +2543,6 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3389,7 +2670,6 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3490,7 +2770,6 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3579,7 +2858,6 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3709,7 +2987,6 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3872,7 +3149,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 m.FloatValue = float64(math.Float64frombits(v)) default: @@ -3887,7 +3164,6 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -3969,7 +3245,6 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4078,17 +3353,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Shards) == 0 { - m.Shards = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4202,7 +3466,6 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4344,7 +3607,6 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4579,17 +3841,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4687,7 +3938,6 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -4844,17 +4094,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.RowIDs) == 0 { - m.RowIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4917,17 +4156,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -4990,17 +4218,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Timestamps) == 0 { - m.Timestamps = make([]int64, 0, elementCount) - } for iNdEx < postIndex { var v int64 for shift := uint(0); ; shift += 7 { @@ -5092,7 +4309,6 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5249,17 +4465,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.ColumnIDs) == 0 { - m.ColumnIDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5322,17 +4527,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Values) == 0 { - m.Values = make([]int64, 0, elementCount) - } for iNdEx < postIndex { var v int64 for shift := uint(0); ; shift += 7 { @@ -5395,7 +4589,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5533,7 +4726,6 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5613,17 +4805,6 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.IDs) == 0 { - m.IDs = make([]uint64, 0, elementCount) - } for iNdEx < postIndex { var v uint64 for shift := uint(0); ; shift += 7 { @@ -5657,7 +4838,6 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5768,7 +4948,6 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5870,7 +5049,6 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -5985,9 +5163,9 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_f65cfea24ac19f54) } +func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } -var fileDescriptor_public_f65cfea24ac19f54 = []byte{ +var fileDescriptorPublic = []byte{ // 880 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x8e, 0x1b, 0x45, 0x10, 0xa6, 0x3d, 0x63, 0x7b, 0x5c, 0x5e, 0x9b, 0xa8, 0xe5, 0x84, 0x11, 0x8a, 0x8c, 0x35, 0x42, diff --git a/logger/logger.go b/logger/logger.go index ed5a2dc26..9e895d482 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -15,6 +15,7 @@ package logger import ( + "fmt" "io" "log" ) @@ -82,3 +83,25 @@ func (vb *verboseLogger) Debugf(format string, v ...interface{}) { func (vb *verboseLogger) Logger() *log.Logger { return vb.logger } + +// CaptureLogger is a logger that stores all the print and debug messages +// it sees, useful for testing. +type CaptureLogger struct { + Prints []string + Debugs []string +} + +// NewCaptureLogger yields a CaptureLogger. +func NewCaptureLogger() *CaptureLogger { + return &CaptureLogger{} +} + +// Printf formats a message and appends it to Prints. +func (cl *CaptureLogger) Printf(format string, v ...interface{}) { + cl.Prints = append(cl.Prints, fmt.Sprintf(format, v...)) +} + +// Debugf formats a message and appends it to Debugs. +func (cl *CaptureLogger) Debugf(format string, v ...interface{}) { + cl.Debugs = append(cl.Debugs, fmt.Sprintf(format, v...)) +} diff --git a/lru/lru.go b/lru/lru.go index ba0121a9a..7f2e6dc22 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -83,7 +83,7 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { } // remove removes the provided key from the cache. -func (c *Cache) remove(key Key) { // nolint: staticcheck +func (c *Cache) remove(key Key) { // nolint: staticcheck,unused if c.cache == nil { return } @@ -121,7 +121,7 @@ func (c *Cache) Len() int { } // clear purges all stored items from the cache. -func (c *Cache) clear() { // nolint: staticcheck +func (c *Cache) clear() { // nolint: staticcheck,unused if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) diff --git a/mock/mock.go b/mock/mock.go index 97ebf8641..60fdfd200 100644 --- a/mock/mock.go +++ b/mock/mock.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package mock import "sync" diff --git a/mock/translator.go b/mock/translator.go index 7a63d8cf0..ca5ef1e8e 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package mock import ( diff --git a/pilosa.go b/pilosa.go index f8989e9b7..42ab3d3c1 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,8 +16,9 @@ package pilosa import ( "encoding/json" - "errors" "regexp" + + "github.com/pkg/errors" ) // System errors. @@ -47,7 +48,7 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 64 characters") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. @@ -76,8 +77,8 @@ type apiMethodNotAllowedError struct { error } -// newApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. -func newApiMethodNotAllowedError(err error) apiMethodNotAllowedError { +// newAPIMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. +func newAPIMethodNotAllowedError(err error) apiMethodNotAllowedError { return apiMethodNotAllowedError{err} } @@ -127,6 +128,8 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } +// MarshalJSON marshals the ColumnAttrSet to JSON such that +// either a Key or an ID is included. func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) { if cas.Key != "" { return json.Marshal(struct { @@ -149,10 +152,10 @@ func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// validateName ensures that the name is a valid format. +// validateName ensures that the index or field name is a valid format. func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { - return ErrName + return errors.Wrapf(ErrName, "'%s'", name) } return nil } diff --git a/pql/ast.go b/pql/ast.go index d78ad2828..49503474c 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -103,7 +103,9 @@ func (q *Query) endConditional() { func (q *Query) addField(field string) { elem := q.lastCallStackElem() - if elem == nil || elem.lastField != "" { + if elem == nil { + panic(fmt.Sprintf("addField called with '%s' while element is nil", field)) + } else if elem.lastField != "" { panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField)) } elem.lastField = field @@ -112,6 +114,15 @@ func (q *Query) addField(field string) { } } +// validateArgField ensures that field does not already +// exist as a key in the Args map before adding the new +// key/value. +func (q *Query) validateArgField(elem *callStackElem) { + if _, exists := elem.call.Args[elem.lastField]; exists { + panic(fmt.Sprintf("%s: %s", duplicateArgErrorMessage, elem.lastField)) + } +} + func (q *Query) addVal(val interface{}) { elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { @@ -123,11 +134,13 @@ func (q *Query) addVal(val interface{}) { return } if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 1 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: val, } } else { + q.validateArgField(elem) // case 2 elem.call.Args[elem.lastField] = val } elem.lastField = "" @@ -147,7 +160,7 @@ func (q *Query) addNumVal(val string) { ival, err = strconv.ParseInt(val, 10, 64) } if err != nil { - panic(err) + panic(fmt.Sprintf("%s: %s", intOutOfRangeError, err)) } if elem.inList { if elem.lastCond != ILLEGAL { @@ -162,11 +175,13 @@ func (q *Query) addNumVal(val string) { } return } else if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 3 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: ival, } } else { + q.validateArgField(elem) // case 4 elem.call.Args[elem.lastField] = ival } elem.lastField = "" @@ -175,6 +190,7 @@ func (q *Query) addNumVal(val string) { func (q *Query) startList() { elem := q.lastCallStackElem() + q.validateArgField(elem) // case 5 if elem.lastCond != ILLEGAL { elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, diff --git a/pql/parser.go b/pql/parser.go index 611294971..e733038d7 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,6 +15,7 @@ package pql import ( + "fmt" "io" "io/ioutil" "strings" @@ -25,6 +26,10 @@ import ( // timeFormat is the go-style time format used to parse string dates. const timeFormat = "2006-01-02T15:04" +// error strings in the parser +const duplicateArgErrorMessage = "duplicate argument provided" +const intOutOfRangeError = "integer is not in signed 64-bit range" + // parser represents a parser for the PQL language. type parser struct { r io.Reader @@ -59,6 +64,24 @@ func (p *parser) Parse() (*Query, error) { if err != nil { return nil, errors.Wrap(err, "parsing") } - p.Execute() + + // Handle specific panics from the parser and return them as errors. + var v interface{} + func() { + defer func() { v = recover() }() + p.Execute() + }() + if v != nil { + errorMessage, ok := v.(string) + if !ok { + return nil, fmt.Errorf("unexpected parser error of type %T: %[1]v", v) + } + if strings.HasPrefix(errorMessage, duplicateArgErrorMessage) || strings.HasPrefix(errorMessage, intOutOfRangeError) { + return nil, fmt.Errorf("%s", v) + } else { + panic(v) + } + } + return &p.Query, nil } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index acec51d19..f1ae962fc 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -15,7 +15,7 @@ const endSymbol rune = 1114112 type pegRule uint8 const ( - ruleUnknown pegRule = iota + ruleUnknown pegRule = iota // nolint:varcheck,deadcode,unused ruleCalls ruleCall ruleallargs diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 8492d3531..3ceff075d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -1,6 +1,21 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( + "fmt" "reflect" "strconv" "testing" @@ -318,6 +333,12 @@ func TestPEGErrors(t *testing.T) { { name: "RangeTimeOneStamp", input: "Row(a=4, 2010-07-04T00:00)"}, + { + name: "ArgOutOfBounds", + input: "Row(a=9223372036854775808)"}, + { + name: "ArgOutOfBoundsNeg", + input: "Row(a=-9223372036854775809)"}, } for i, test := range tests { @@ -676,3 +697,47 @@ func TestPQLDeepEquality(t *testing.T) { }) } } + +func TestDuplicateArgError(t *testing.T) { + tests := []struct { + name string + call string + }{ + // case 1 + { + name: "StringConditional", + call: "Row(a==foo, a==bar)", + }, + // case 2 + { + name: "StringValue", + call: "Row(a=foo, a=bar)", + }, + // case 3 + { + name: "IntConditional", + call: "Row(a>5, a>6)", + }, + // case 4 + { + name: "IntValue", + call: "Row(a=7, a=8)", + }, + // case 5 + { + name: "List", + call: "Row(a=[7], a=[7,8])", + }, + } + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + _, err := ParseString(test.call) + expErr := fmt.Sprintf("%s: a", duplicateArgErrorMessage) + if err == nil { + t.Fatalf("expected error for duplicate argument: %s", test.call) + } else if err.Error() != expErr { + t.Fatalf("expected error: %s, but got: %v", expErr, err.Error()) + } + }) + } +} diff --git a/roaring/btree.go b/roaring/btree.go index 76427059f..53489af89 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -570,6 +570,13 @@ func (t *tree) Set(k uint64, v *Container) { //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) //}() + // we don't want to store nil containers; if you try to set a + // container to nil, that's equivalent to not having one at that + // location. + if v == nil { + _ = t.Delete(k) + return + } pi := -1 var p *x @@ -642,7 +649,9 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } - + if newV == nil { + return + } z := t.insert(btDPool.Get().(*d), 0, k, newV) t.r, t.first, t.last = z, z, z return @@ -667,7 +676,11 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } - + // delete nil containers rather than storing them. + if newV == nil { + t.Delete(k) + return + } x.d[i].v = newV } return @@ -686,6 +699,10 @@ func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Conta if !written { return } + // nil values don't need to exist, and break iteration later. + if newV == nil { + return + } switch { case x.c < 2*kd: @@ -870,8 +887,50 @@ func (e *enumerator) Next() (k uint64, v *Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.next() - return k, v, nil + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.next() + return k, v, err +} + +// Every iterates over a tree. +func (e *enumerator) Every(upd func(oldV *Container, exists bool) (newV *Container, write bool)) error { + if err := e.err; err != nil { + return err + } + + if e.ver != e.t.ver { + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + } + + for { + if e.q == nil { + e.err = io.EOF + return e.err + } + + if e.i >= e.q.c { + if err := e.next(); err != nil { + e.err = err + return e.err + } + } + + i := e.q.d[e.i] + nv, write := upd(i.v, true) + if write { + if nv == nil { + e.t.Delete(e.q.d[e.i].k) + } else { + e.q.d[e.i].v = nv + } + } + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.next() + } } func (e *enumerator) next() error { @@ -925,7 +984,9 @@ func (e *enumerator) Prev() (k uint64, v *Container, err error) { i := e.q.d[e.i] k, v = i.k, i.v e.k, e.hit = k, true - e.prev() + // Any error returned would be stashed in e.err, and would come up + // on the next call. + _ = e.prev() return k, v, err } diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 67fd49e68..422a7b767 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -101,16 +101,16 @@ func (t *tree) dump() string { n = i + 1 } } - f.Format("%sX#%d(%p) n %d:%d {", pref, h, x, x.c, n) + _, _ = f.Format("%sX#%d(%p) n %d:%d {", pref, h, x, x.c, n) a := []interface{}{} for i, v := range x.x[:n] { a = append(a, v.ch) if i != 0 { - f.Format(" ") + _, _ = f.Format(" ") } - f.Format("(C#%d K %v)", handle(v.ch), v.k) + _, _ = f.Format("(C#%d K %v)", handle(v.ch), v.k) } - f.Format("}\n") + _, _ = f.Format("}\n") for _, p := range a { pagedump(p, pref+". ") } @@ -122,14 +122,14 @@ func (t *tree) dump() string { n = i + 1 } } - f.Format("%sD#%d(%p) P#%d N#%d n %d:%d {", pref, h, x, handle(x.p), handle(x.n), x.c, n) + _, _ = f.Format("%sD#%d(%p) P#%d N#%d n %d:%d {", pref, h, x, handle(x.p), handle(x.n), x.c, n) for i, d := range x.d[:n] { if i != 0 { - f.Format(" ") + _, _ = f.Format(" ") } - f.Format("%v:%v", d.k, d.v) + _, _ = f.Format("%v:%v", d.k, d.v) } - f.Format("}\n") + _, _ = f.Format("}\n") } } @@ -467,13 +467,14 @@ func benchmarkSetRnd(b *testing.B, n int) { a[i] = rng.Next() } b.ResetTimer() + c := getDummyC(1) for i := 0; i < b.N; i++ { b.StopTimer() r := treeNew() debug.FreeOSMemory() b.StartTimer() for _, v := range a { - r.Set(uint64(v), nil) + r.Set(uint64(v), c) } b.StopTimer() r.Close() @@ -504,8 +505,9 @@ func benchmarkGetRnd(b *testing.B, n int) { for i := range a { a[i] = rng.Next() } + c := getDummyC(1) for _, v := range a { - r.Set(uint64(v), nil) + r.Set(uint64(v), c) } debug.FreeOSMemory() b.ResetTimer() @@ -1392,7 +1394,7 @@ func TestBtreePut(t *testing.T) { t.Fatal(iTest, g, e) } } - return nil, test.write + return getDummyC(99), test.write }) if test.exists { if g, e := oldV, getDummyC(test.oldV); g != e { @@ -1427,6 +1429,8 @@ func TestBtreePut(t *testing.T) { var e *Container if test.post[i+1] != -1 { e = getDummyC(test.post[i+1]) + } else { + e = getDummyC(99) } if g := v; g != e { t.Fatal(iTest, g, e) @@ -1445,7 +1449,7 @@ func TestBtreeSeek(t *testing.T) { tr := treeNew() for i := 0; i < N; i++ { k := 2*i + 1 - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) } for i := 0; i < N; i++ { k := 2 * i @@ -1476,14 +1480,14 @@ func TestBtreePR4(t *testing.T) { tr := treeNew() for i := 0; i < 2*kd+1; i++ { k := 1000 * i - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) } tr.Delete(1000 * kd) for i := 0; i < kd; i++ { - tr.Set(uint64(1000*(kd+1)-1-i), nil) + tr.Set(uint64(1000*(kd+1)-1-i), getDummyC(1)) } k := 1000*(kd+1) - 1 - kd - tr.Set(uint64(k), nil) + tr.Set(uint64(k), getDummyC(1)) if _, ok := tr.Get(uint64(k)); !ok { t.Fatalf("key lost: %v", k) } diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 9d3f82302..fe03f9d2e 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -1,6 +1,21 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import ( + "fmt" "reflect" "runtime" "unsafe" @@ -25,24 +40,84 @@ type Container struct { pointer *uint16 // the data pointer len, cap int32 // length and cap n int32 // number of integers in container - mapped bool // mapped directly to a byte slice when true - typ byte // array, bitmap, or run + flags containerFlags // internal flags + typeID byte // array, bitmap, or run data [stashedArraySize]uint16 // immediate data for small arrays or runs } +type containerFlags uint8 + +const ( + flagMapped = containerFlags(1 << iota) + flagFrozen +) + +func (c *Container) String() string { + if c == nil { + return "" + } + froze := "" + switch c.flags { + case flagFrozen: + froze = "frozen " + case flagMapped: + froze = "mapped " + case flagFrozen | flagMapped: + froze = "frozen/mapped" + } + switch c.typeID { + case containerArray: + return fmt.Sprintf("<%sarray container, N=%d>", froze, c.N()) + case containerBitmap: + return fmt.Sprintf("<%sbitmap container, N=%d, len %dx uint64>", + froze, c.N(), len(c.bitmap())) + case containerRun: + return fmt.Sprintf("<%srun container, N=%d, len %dx interval>", + froze, c.N(), len(c.runs())) + default: + return fmt.Sprintf("", froze, c.typeID, c.N()) + } +} + // NewContainer returns a new instance of container. This trivial function // may later become more interesting. func NewContainer() *Container { statsHit("NewContainer") - c := &Container{typ: containerArray, len: 0, cap: stashedArraySize} + c := &Container{typeID: containerArray, len: 0, cap: stashedArraySize} c.pointer = (*uint16)(unsafe.Pointer(&c.data[0])) return c } // NewContainerBitmap makes a bitmap container using the provided bitmap, or // an empty one if provided bitmap is nil. If the provided bitmap is too short, -// it will be padded. -func NewContainerBitmap(n int32, bitmap []uint64) *Container { +// it will be padded. This function's API is wrong; it should have been +// written as NewContainerBitmapN, and this should not take the n argument, +// but I did it wrong initially and now that would be a breaking change. +func NewContainerBitmap(n int, bitmap []uint64) *Container { + if bitmap == nil { + return NewContainerBitmapN(nil, 0) + } + // pad to required length + if len(bitmap) < bitmapN { + bm2 := make([]uint64, bitmapN) + copy(bm2, bitmap) + bitmap = bm2 + } + c := &Container{typeID: containerBitmap} + c.setBitmap(bitmap) + // set n based on bitmap contents. + if n < 0 { + c.bitmapRepair() + } else { + c.setN(int32(n)) + } + return c +} + +// NewContainerBitmapN makes a bitmap container using the provided bitmap, or +// an empty one if provided bitmap is nil. If the provided bitmap is too short, +// it will be padded. The container's count is specified directly. +func NewContainerBitmapN(bitmap []uint64, n int32) *Container { if bitmap == nil { bitmap = make([]uint64, bitmapN) } @@ -52,23 +127,40 @@ func NewContainerBitmap(n int32, bitmap []uint64) *Container { copy(bm2, bitmap) bitmap = bm2 } - c := &Container{typ: containerBitmap, n: n} + c := &Container{typeID: containerBitmap, n: n} c.setBitmap(bitmap) return c } -// NewContainerArray returns an array using the provided set of values. It's -// okay if the slice is nil; that's a length of zero. +// NewContainerArray returns an array container using the provided set of +// values. It's okay if the slice is nil; that's a length of zero. func NewContainerArray(set []uint16) *Container { - c := &Container{typ: containerArray, n: int32(len(set))} + c := &Container{typeID: containerArray, n: int32(len(set))} c.setArray(set) return c } -// NewContainerRun creates a new run array using a provided (possibly nil) +// NewContainerArrayCopy returns an array container using the provided set of +// values. It's okay if the slice is nil; that's a length of zero. It copies +// the provided slice to new storage. +func NewContainerArrayCopy(set []uint16) *Container { + c := &Container{typeID: containerArray, n: int32(len(set))} + c.setArrayMaybeCopy(set, true) + return c +} + +// NewContainerArrayN returns an array container using the specified +// set of values, but overriding n. +func NewContainerArrayN(set []uint16, n int32) *Container { + c := &Container{typeID: containerArray, n: n} + c.setArray(set) + return c +} + +// NewContainerRun creates a new run container using a provided (possibly nil) // slice of intervals. func NewContainerRun(set []interval16) *Container { - c := &Container{typ: containerRun} + c := &Container{typeID: containerRun} c.setRuns(set) for _, run := range set { c.n += int32(run.last-run.start) + 1 @@ -76,61 +168,251 @@ func NewContainerRun(set []interval16) *Container { return c } +// NewContainerRunCopy creates a new run container using a provided (possibly nil) +// slice of intervals. It copies the provided slice to new storage. +func NewContainerRunCopy(set []interval16) *Container { + c := &Container{typeID: containerRun} + c.setRunsMaybeCopy(set, true) + for _, run := range set { + c.n += int32(run.last-run.start) + 1 + } + return c +} + +// NewContainerRunN creates a new run array using a provided (possibly nil) +// slice of intervals. It overrides n using the provided value. +func NewContainerRunN(set []interval16, n int32) *Container { + c := &Container{typeID: containerRun, n: n} + c.setRuns(set) + return c +} + // Mapped returns the internal mapped field, which indicates whether the // slice's backing store is believed to be associated with unwriteable // mmapped space. func (c *Container) Mapped() bool { - return c.mapped + if c == nil { + return false + } + return (c.flags & flagMapped) != 0 } -// N returns the internal n field. +// frozen() returns the internal frozen state. It isn't exported because +// nothing outside this package should be thinking about this. +func (c *Container) frozen() bool { + if c == nil { + return true + } + return (c.flags & flagFrozen) != 0 +} + +// N returns the 1-count of the container. func (c *Container) N() int32 { + if c == nil { + return 0 + } return c.n } +func (c *Container) setN(n int32) { + if c == nil { + if roaringParanoia { + panic("trying to setN on a nil container") + } + return + } + c.n = n +} + +func (c *Container) typ() byte { + if c == nil { + return containerNil + } + return c.typeID +} + +// setTyp should only be called if you already know that c is a +// non-nil, non-frozen, container. +func (c *Container) setTyp(newType byte) { + if roaringParanoia { + if c == nil || c.frozen() { + panic("setTyp on nil or frozen container") + } + } + c.typeID = newType +} + +func (c *Container) setMapped(mapped bool) { + if roaringParanoia { + if c == nil || c.frozen() { + panic("setMapped on nil or frozen container") + } + } + if mapped { + c.flags |= flagMapped + } else { + c.flags &^= flagMapped + } +} + +// Freeze returns an unmodifiable container identical to c. This might +// be c, now marked unmodifiable, or might be a new container. If c +// is currently marked as "mapped", referring to a backing store that's +// not a conventional Go pointer, the storage may be copied. +func (c *Container) Freeze() *Container { + if c == nil { + return nil + } + // don't need to freeze + if c.flags&flagFrozen != 0 { + return c + } + // unmapOrClone should unmap-in-place because the existing + // container isn't frozen (or we'd already have returned it). + c = c.unmapOrClone() + c.flags |= flagFrozen + return c +} + +// Thaw returns a modifiable container identical to c. This may be c, or it +// may be a new container with distinct backing store. +func (c *Container) Thaw() *Container { + if roaringParanoia { + if c == nil { + panic("trying to thaw a nil container") + } + } + if c.flags&(flagFrozen|flagMapped) == 0 { + return c + } + return c.unmapOrClone() +} + +func (c *Container) unmapOrClone() *Container { + if c.flags&flagFrozen != 0 { + // Caqn't modify this container, therefore, we have to make a + // copy. + return c.Clone() + } + c.flags &^= flagMapped + // mapped: we want to unmap the storage. + switch c.typeID { + case containerArray: + // mapped flag is wrong here + if c.pointer == (*uint16)(unsafe.Pointer(&c.data)) { + return c + } + // maybe it fits in storage + if c.len <= stashedArraySize { + copy(c.data[:stashedArraySize], c.array()) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(&c.data)), stashedArraySize + return c + } + array := c.array() + tmp := make([]uint16, c.len) + copy(tmp, array) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) + runtime.KeepAlive(&tmp) + case containerRun: + // mapped flag is wrong here + if c.pointer == (*uint16)(unsafe.Pointer(&c.data)) { + return c + } + oldRuns := c.runs() + // maybe it fits in storage + if c.len <= stashedRunSize { + c.pointer, c.cap = (*uint16)(unsafe.Pointer(&c.data)), stashedRunSize + copy(c.runs(), oldRuns) + return c + } + tmp := make([]interval16, c.len) + copy(tmp, oldRuns) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) + runtime.KeepAlive(&tmp) + case containerBitmap: + bitmap := c.bitmap() + tmp := make([]uint64, bitmapN) + copy(tmp, bitmap) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), bitmapN, bitmapN + runtime.KeepAlive(&tmp) + default: + panic(fmt.Sprintf("can't thaw invalid container, type %d", c.typeID)) + } + return c +} + // array yields the data viewed as a slice of uint16 values. func (c *Container) array() []uint16 { if roaringParanoia { - if c.typ != containerArray { + if c == nil { + panic("attempt to read a nil container's array") + } + if c.typeID != containerArray { panic("attempt to read non-array's array") } } return *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) } -// setArray stores a set of uint16s as data. -func (c *Container) setArray(array []uint16) { +// setArrayMaybeCopy stores a set of uint16s as data. c must not be frozen. +// If doCopy is set, it will ensure that the data get copied (possibly to +// its internal stash.) +func (c *Container) setArrayMaybeCopy(array []uint16, doCopy bool) { if roaringParanoia { - if c.typ != containerArray { + if c == nil || c.frozen() { + panic("setArray on nil or frozen container") + } + if c.typeID != containerArray { panic("attempt to write non-array's array") } } // no array: start with our default 5-value array if array == nil { c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedArraySize + c.n = c.len return } h := (*reflect.SliceHeader)(unsafe.Pointer(&array)) if h.Data == uintptr(unsafe.Pointer(c.pointer)) { // nothing to do but update length c.len = int32(h.Len) + c.n = c.len return } // array we can fit in data store: if len(array) <= stashedArraySize { copy(c.data[:stashedArraySize], array) c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(array)), stashedArraySize - c.mapped = false // this is no longer using a hypothetical mmapped input array + c.n = c.len + c.flags &^= flagMapped // this is no longer using a hypothetical mmapped input array return } + // copy the array + if doCopy { + a2 := make([]uint16, len(array)) + copy(a2, array) + h = (*reflect.SliceHeader)(unsafe.Pointer(&a2)) + } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) + c.n = c.len runtime.KeepAlive(&array) } +// setArrayMaybeCopy stores a set of uint16s as data. c must not be frozen. +func (c *Container) setArray(array []uint16) { + c.setArrayMaybeCopy(array, false) +} + // bitmap yields the data viewed as a slice of uint64s holding bits. func (c *Container) bitmap() []uint64 { if roaringParanoia { - if c.typ != containerBitmap { + if c == nil { + panic("attempt to read nil container's bitmap") + } + if c.typeID != containerBitmap { panic("attempt to read non-bitmap's bitmap") } } @@ -139,8 +421,11 @@ func (c *Container) bitmap() []uint64 { // setBitmap stores a set of uint64s as data. func (c *Container) setBitmap(bitmap []uint64) { + if c == nil || c.frozen() { + panic("setBitmap on nil or frozen container") + } if roaringParanoia { - if c.typ != containerBitmap { + if c.typeID != containerBitmap { panic("attempt to write non-bitmap's bitmap") } } @@ -152,17 +437,29 @@ func (c *Container) setBitmap(bitmap []uint64) { // runs yields the data viewed as a slice of intervals. func (c *Container) runs() []interval16 { if roaringParanoia { - if c.typ != containerRun { + if c == nil { + panic("attempt to read nil container's runs") + } + if c.typeID != containerRun { panic("attempt to read non-run's runs") } } return *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) } -// setRuns stores a set of intervals as data. +// setRuns stores a set of intervals as data. c must not be frozen. func (c *Container) setRuns(runs []interval16) { + c.setRunsMaybeCopy(runs, false) +} + +// setRunsMaybeCopy stores a set of intervals as data. c must not be frozen. +// If doCopy is set, the values will be copied to different storage. +func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) { if roaringParanoia { - if c.typ != containerRun { + if c == nil || c.frozen() { + panic("setRuns on nil or frozen container") + } + if c.typeID != containerRun { panic("attempt to write non-run's runs") } } @@ -183,20 +480,64 @@ func (c *Container) setRuns(runs []interval16) { newRuns := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&c.data[0])), Len: stashedRunSize, Cap: stashedRunSize})) copy(newRuns, runs) c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(runs)), stashedRunSize - c.mapped = false // this is no longer using a hypothetical mmapped input array + c.flags &^= flagMapped // this is no longer using a hypothetical mmapped input array return } + if doCopy { + r2 := make([]interval16, len(runs)) + copy(r2, runs) + h = (*reflect.SliceHeader)(unsafe.Pointer(&r2)) + } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) runtime.KeepAlive(&runs) } -// Update updates the container -func (c *Container) Update(typ byte, n int32, mapped bool) { - c.typ = typ +// UpdateOrMake updates the container, yielding a new container if necessary. +func (c *Container) UpdateOrMake(typ byte, n int32, mapped bool) *Container { + if c == nil { + switch typ { + case containerRun: + c = NewContainerRunN(nil, n) + case containerBitmap: + c = NewContainerBitmapN(nil, n) + default: + c = NewContainerArrayN(nil, n) + } + c.flags |= flagMapped + return c + } + // ensure that we are allowed to modify this container + c = c.Thaw() + c.typeID = typ c.n = n - c.mapped = mapped + // note: this probably shouldn't be happening, the decision should be getting + // made when we specify the storage. + c.setMapped(mapped) // we don't know that any existing slice is usable, so let's ditch it - switch c.typ { + switch c.typeID { + case containerArray: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(0), stashedArraySize + case containerRun: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedRunSize + default: + c.pointer, c.len, c.cap = nil, 0, 0 + } + return c +} + +// Update updates the container if possible. It is an error to +// call Update on a frozen container. +func (c *Container) Update(typ byte, n int32, mapped bool) { + if c == nil || c.frozen() { + panic("cannot Update a nil or frozen container") + } + c.typeID = typ + c.n = n + // note: this probably shouldn't be happening, the decision should be getting + // made when we specify the storage. + c.setMapped(mapped) + // we don't know that any existing slice is usable, so let's ditch it + switch c.typeID { case containerArray: c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(0), stashedArraySize case containerRun: @@ -208,56 +549,30 @@ func (c *Container) Update(typ byte, n int32, mapped bool) { // isArray returns true if the container is an array container. func (c *Container) isArray() bool { - return c.typ == containerArray + if roaringParanoia { + if c == nil { + panic("calling isArray on nil container") + } + } + return c.typeID == containerArray } // isBitmap returns true if the container is a bitmap container. func (c *Container) isBitmap() bool { - return c.typ == containerBitmap + if roaringParanoia { + if c == nil { + panic("calling isBitmap on nil container") + } + } + return c.typeID == containerBitmap } // isRun returns true if the container is a run-length-encoded container. func (c *Container) isRun() bool { - return c.typ == containerRun -} - -// unmapArray ensures that the container is not using mmapped storage. -func (c *Container) unmapArray() { - if !c.mapped { - return + if roaringParanoia { + if c == nil { + panic("calling isRun on nil container") + } } - array := c.array() - tmp := make([]uint16, c.len) - copy(tmp, array) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - runtime.KeepAlive(&tmp) - c.mapped = false -} - -// unmapBitmap ensures that the container is not using mmapped storage. -func (c *Container) unmapBitmap() { - if !c.mapped { - return - } - bitmap := c.bitmap() - tmp := make([]uint64, c.len) - copy(tmp, bitmap) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - runtime.KeepAlive(&tmp) - c.mapped = false -} - -// unmapRun ensures that the container is not using mmapped storage. -func (c *Container) unmapRun() { - if !c.mapped { - return - } - runs := c.runs() - tmp := make([]interval16, c.len) - copy(tmp, runs) - h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) - c.pointer, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Cap) - c.mapped = false + return c.typeID == containerRun } diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index fb29a6d08..c2dbc01f5 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package roaring import ( + "fmt" "io" ) @@ -35,13 +36,14 @@ func NewBTreeBitmap(a ...uint64) *Bitmap { b := &Bitmap{ Containers: newBTreeContainers(), } - b.Add(a...) + // TODO: We have no way to report this. + _, _ = b.Add(a...) return b } func (btc *bTreeContainers) Get(key uint64) *Container { // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { + if key == btc.lastKey { return btc.lastContainer } @@ -56,11 +58,15 @@ func (btc *bTreeContainers) Get(key uint64) *Container { } func (btc *bTreeContainers) Put(key uint64, c *Container) { + // If we don't do this, a Put on a container we just got from + // Get can result in the tree containing a different container + // than we'll get on next lookup. + btc.lastKey, btc.lastContainer = key, c // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. if c.Mapped() { - btc.lastContainer = nil + btc.lastKey = ^uint64(0) } btc.tree.Set(key, c) } @@ -68,13 +74,13 @@ func (btc *bTreeContainers) Put(key uint64, c *Container) { func (u updater) update(oldV *Container, exists bool) (*Container, bool) { // update the existing container if exists { - oldV.Update(u.typ, u.n, u.mapped) - return oldV, false + oldV = oldV.UpdateOrMake(u.typ, u.n, u.mapped) + return oldV, true } cont := NewContainer() - cont.typ = u.typ - cont.n = u.n - cont.mapped = u.mapped + cont.setTyp(u.typ) + cont.setN(u.n) + cont.setMapped(u.mapped) return cont, true } @@ -97,7 +103,7 @@ func (btc *bTreeContainers) Remove(key uint64) { func (btc *bTreeContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. - if key == btc.lastKey && btc.lastContainer != nil { + if key == btc.lastKey { return btc.lastContainer } @@ -118,7 +124,7 @@ func (btc *bTreeContainers) Count() (n uint64) { e, _ := btc.tree.Seek(0) _, c, err := e.Next() for err != io.EOF { - n += uint64(c.n) + n += uint64(c.N()) _, c, err = e.Next() } return n @@ -141,6 +147,23 @@ func (btc *bTreeContainers) Clone() Containers { return nbtc } +func (btc *bTreeContainers) Freeze() Containers { + nbtc := newBTreeContainers() + + itr, err := btc.tree.SeekFirst() + if err == io.EOF { + return nbtc + } + for { + k, v, err := itr.Next() + if err == io.EOF { + break + } + nbtc.tree.Set(k, v.Freeze()) + } + return nbtc +} + func (btc *bTreeContainers) Last() (key uint64, c *Container) { if btc.tree.Len() == 0 { return 0, nil @@ -155,7 +178,10 @@ func (btc *bTreeContainers) Size() int { func (btc *bTreeContainers) Reset() { btc.tree = treeNew() - btc.lastKey = 0 + // use a definitely-invalid key, so we can distinguish between "you + // just looked that up, and it was a nil container" and "you have + // never looked that up before." + btc.lastKey = ^uint64(0) btc.lastContainer = nil } @@ -179,6 +205,23 @@ func (btc *bTreeContainers) Repair() { } } +// Update calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (btc *bTreeContainers) Update(key uint64, fn func(*Container, bool) (*Container, bool)) { + btc.tree.Put(key, fn) +} + +// UpdateEvery calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (btc *bTreeContainers) UpdateEvery(fn func(*Container, bool) (*Container, bool)) { + e, _ := btc.tree.Seek(0) + // currently not handling the error from this, but in practice it has + // to be io.EOF. + _ = e.Every(fn) +} + type btcIterator struct { e *enumerator key uint64 @@ -186,19 +229,20 @@ type btcIterator struct { } func (i *btcIterator) Next() bool { - k, v, err := i.e.Next() if err == io.EOF { return false } + if roaringParanoia { + if v == nil { + panic(fmt.Sprintf("got nil container for key %d", k)) + } + } i.key = k i.val = v return true } func (i *btcIterator) Value() (uint64, *Container) { - if i.val == nil { - return 0, nil - } return i.key, i.val } diff --git a/roaring/containers.go b/roaring/containers_slice.go similarity index 63% rename from roaring/containers.go rename to roaring/containers_slice.go index cc9a09ec5..cbff4f179 100644 --- a/roaring/containers.go +++ b/roaring/containers_slice.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -50,15 +50,21 @@ func (sc *sliceContainers) PutContainerValues(key uint64, typ byte, n int, mappe i := search64(sc.keys, key) if i < 0 { c := NewContainer() - c.typ = typ - c.n = int32(n) - c.mapped = mapped + c.setTyp(typ) + c.setN(int32(n)) + c.setMapped(mapped) sc.insertAt(key, c, -i-1) } else { - c := sc.containers[i] - c.typ = typ - c.n = int32(n) - c.mapped = mapped + // if the container already exists, and is frozen, this may + // result in copying its data, which is sort of pointless + // because PutContainerValues almost always gets called + // because we're reading new data from a file -- but also + // that means this case probably never happens. + c := sc.containers[i].Thaw() + c.setTyp(typ) + c.setN(int32(n)) + c.setMapped(mapped) + sc.containers[i] = c } } @@ -114,6 +120,17 @@ func (sc *sliceContainers) Clone() Containers { return other } +func (sc *sliceContainers) Freeze() Containers { + other := newSliceContainers() + other.keys = make([]uint64, len(sc.keys)) + other.containers = make([]*Container, len(sc.containers)) + copy(other.keys, sc.keys) + for i, c := range sc.containers { + other.containers[i] = c.Freeze() + } + return other +} + func (sc *sliceContainers) Last() (key uint64, c *Container) { if len(sc.keys) == 0 { return 0, nil @@ -129,7 +146,7 @@ func (sc *sliceContainers) Size() int { func (sc *sliceContainers) Count() uint64 { n := uint64(0) for i := range sc.containers { - n += uint64(sc.containers[i].n) + n += uint64(sc.containers[i].N()) } return n } @@ -162,6 +179,42 @@ func (sc *sliceContainers) Repair() { } } +// Update calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (sc *sliceContainers) Update(key uint64, fn func(*Container, bool) (*Container, bool)) { + i, found := sc.seek(key) + var nc *Container + var write bool + if found { + nc, write = fn(sc.containers[i], true) + if write { + sc.containers[i] = nc + } + } else { + nc, write = fn(nil, false) + // don't expand the slice just to add a nil container, we + // could return that anyway + if write && nc != nil { + sc.containers = append(sc.containers, nil) + copy(sc.containers[i+1:], sc.containers[i:]) + sc.containers[i] = nc + } + } +} + +// UpdateEvery calls fn (existing-container, existed), and expects +// (new-container, write). If write is true, the container is used to +// replace the given container. +func (sc *sliceContainers) UpdateEvery(fn func(*Container, bool) (*Container, bool)) { + for i, c := range sc.containers { + nc, write := fn(c, true) + if write { + sc.containers[i] = nc + } + } +} + type sliceIterator struct { e *sliceContainers i int @@ -170,14 +223,20 @@ type sliceIterator struct { } func (si *sliceIterator) Next() bool { - if si.e == nil || si.i > len(si.e.keys)-1 { + if si.e == nil { return false } - si.key = si.e.keys[si.i] - si.value = si.e.containers[si.i] - si.i++ - - return true + // discard nil containers from iteration. we don't always + // actually remove them because copying is expensive. + for si.i < len(si.e.keys) { + si.key = si.e.keys[si.i] + si.value = si.e.containers[si.i] + si.i++ + if si.value != nil { + return true + } + } + return false } func (si *sliceIterator) Value() (uint64, *Container) { diff --git a/roaring/containers_test.go b/roaring/containers_test.go index 9c8bedd22..abe5dbd99 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -43,14 +43,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !itr.Next() { t.Fatalf("one should be next, but got false") } - if key, val := itr.Value(); key != 1 || val.n != 1 { - t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 1 || val.N() != 1 { + t.Fatalf("Wrong k/v, exp: 1,1 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("two should be next, but got false") } - if key, val := itr.Value(); key != 2 || val.n != 2 { - t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 2 || val.N() != 2 { + t.Fatalf("Wrong k/v, exp: 2,2 got: %v,%v", key, val.N()) } if itr.Next() { @@ -68,14 +68,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !found { t.Fatalf("should have found 3") } - if key, val := itr.Value(); key != 3 || val.n != 3 { - t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 3 || val.N() != 3 { + t.Fatalf("Wrong k/v, exp: 3,3 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("5 should be next, but got false") } - if key, val := itr.Value(); key != 5 || val.n != 5 { - t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 5 || val.N() != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.N()) } itr, found = cs.Iterator(4) @@ -85,14 +85,14 @@ func testContainersIterator(cs Containers, t *testing.T) { if !itr.Next() { t.Fatalf("5 should be next, but got false") } - if key, val := itr.Value(); key != 5 || val.n != 5 { - t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 5 || val.N() != 5 { + t.Fatalf("Wrong k/v, exp: 5,5 got: %v,%v", key, val.N()) } if !itr.Next() { t.Fatalf("6 should be next, but got false") } - if key, val := itr.Value(); key != 6 || val.n != 6 { - t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.n) + if key, val := itr.Value(); key != 6 || val.N() != 6 { + t.Fatalf("Wrong k/v, exp: 6,6 got: %v,%v", key, val.N()) } if itr.Next() { diff --git a/roaring/inst.go b/roaring/inst.go index c0a5771d9..f9eaf2831 100644 --- a/roaring/inst.go +++ b/roaring/inst.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build btreeInstrumentation package roaring diff --git a/roaring/nop_inst.go b/roaring/nop_inst.go index 7a945cb6d..b5cd94458 100644 --- a/roaring/nop_inst.go +++ b/roaring/nop_inst.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !btreeInstrumentation package roaring diff --git a/roaring/roaring.go b/roaring/roaring.go index a8ccd81e2..f09667104 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -31,16 +31,18 @@ const ( // MagicNumber is an identifier, in bytes 0-1 of the file. MagicNumber = uint32(12348) - // storageVersion indicates the storage version, in bytes 2-3. + // storageVersion indicates the storage version, in byte 2. storageVersion = uint32(0) - // cookie is the first four bytes in a roaring bitmap file, + // NOTE: byte 3 stores user-defined flags. + + // cookie is the first 3 bytes in a roaring bitmap file, // formed by joining MagicNumber and storageVersion cookie = MagicNumber + storageVersion<<16 - // headerBaseSize is the size in bytes of the cookie and key count at the - // beginning of a file. - headerBaseSize = 4 + 4 + // headerBaseSize is the size in bytes of the cookie, flags, and key count + // at the beginning of a file. + headerBaseSize = 3 + 1 + 4 // runCountHeaderSize is the size in bytes of the run count stored // at the beginning of every serialized run container. @@ -60,9 +62,10 @@ const ( ) const ( - containerArray byte = iota + 1 // slice of bit position values - containerBitmap // slice of 1024 uint64s - containerRun // container of run-encoded bits + containerNil byte = iota // no container + containerArray // slice of bit position values + containerBitmap // slice of 1024 uint64s + containerRun // container of run-encoded bits ) // map used for a more descriptive print @@ -72,6 +75,8 @@ var containerTypeNames = map[byte]string{ containerRun: "run", } +var fullContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}).Freeze() + type Containers interface { // Get returns nil if the key does not exist. Get(key uint64) *Container @@ -93,12 +98,27 @@ type Containers interface { // Clone does a deep copy of Containers, including cloning all containers contained. Clone() Containers + // Freeze creates a shallow copy of Containers, freezing all the containers + // contained. The new copy is a distinct Containers, but the individual containers + // are shared (but marked as frozen). + Freeze() Containers + // Last returns the highest key and associated container. Last() (key uint64, c *Container) // Size returns the number of containers stored. Size() int + // Update calls fn (existing-container, existed), and expects + // (new-container, write). If write is true, the container is used to + // replace the given container. + Update(key uint64, fn func(*Container, bool) (*Container, bool)) + + // UpdateEvery calls fn (existing-container, existed), and expects + // (new-container, write). If write is true, the container is used to + // replace the given container. + UpdateEvery(fn func(*Container, bool) (*Container, bool)) + // Iterator returns a Contiterator which after a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a // container is found at key. @@ -123,6 +143,9 @@ type ContainerIterator interface { type Bitmap struct { Containers Containers + // User-defined flags. + Flags byte + // Number of bit change operations written to the writer. Some operations // contain multiple values, each of those counts the number of values rather // than counting as one operation. @@ -137,7 +160,25 @@ func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ Containers: newSliceContainers(), } - b.AddN(a...) + // TODO: We have no way to report this. We aren't in a server context + // so we haven't got a logger, nothing is checking for nil returns + // from this... + _, _ = b.AddN(a...) + return b +} + +// NewSliceBitmap makes a new bitmap, explicitly selecting the slice containers +// type, which performs better in cases where we expect a contiguous block of +// containers added in ascending order, such as when extracting a range from +// another bitmap. +func NewSliceBitmap(a ...uint64) *Bitmap { + b := &Bitmap{ + Containers: newSliceContainers(), + } + // TODO: We have no way to report this. We aren't in a server context + // so we haven't got a logger, nothing is checking for nil returns + // from this... + _, _ = b.AddN(a...) return b } @@ -160,6 +201,23 @@ func (b *Bitmap) Clone() *Bitmap { return other } +// Freeze returns a shallow copy of the bitmap. The new bitmap +// is a distinct bitmap, with a new Containers object, but the +// actual containers it holds are the same as the parent's +// containers, but have been frozen. +func (b *Bitmap) Freeze() *Bitmap { + if b == nil { + return nil + } + + // Create a copy of the bitmap structure. + other := &Bitmap{ + Containers: b.Containers.Freeze(), + } + + return other +} + // Add adds values to the bitmap. TODO(2.0) deprecate - use the more general // AddN (though be aware that it modifies 'a' in place). func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { @@ -227,7 +285,7 @@ func (b *Bitmap) DirectRemoveN(a ...uint64) (changed int) { // container level operation across a list of values and return the number of // trues while modifying the list of values in place to contain the // true-returning values in order. -func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (changed int) { +func (b *Bitmap) directOpN(op func(c *Container, v uint16) (*Container, bool), a ...uint64) (changed int) { hb := uint64(0xFFFFFFFFFFFFFFFF) // impossible sentinel value var cont *Container for _, v := range a { @@ -235,10 +293,15 @@ func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (c hb = newhb cont = b.Containers.GetOrCreate(hb) } - if op(cont, lowbits(v)) { + newC, change := op(cont, lowbits(v)) + if change { a[changed] = v changed++ } + if newC != cont { + b.Containers.Put(hb, newC) + cont = newC + } } return changed } @@ -247,7 +310,11 @@ func (b *Bitmap) directOpN(op func(c *Container, v uint16) bool, a ...uint64) (c // deprecate in favor of DirectAddN. func (b *Bitmap) DirectAdd(v uint64) bool { cont := b.Containers.GetOrCreate(highbits(v)) - return cont.add(lowbits(v)) + newC, changed := cont.add(lowbits(v)) + if newC != cont { + b.Containers.Put(highbits(v), newC) + } + return changed } // Contains returns true if v is in the bitmap. @@ -305,11 +372,11 @@ func (b *Bitmap) RemoveN(a ...uint64) (changed int, err error) { func (b *Bitmap) remove(v uint64) bool { c := b.Containers.Get(highbits(v)) - if c == nil { - return false + newC, changed := c.remove(lowbits(v)) + if newC != c { + b.Containers.Put(highbits(v), newC) } - // TODO - do nil check inside c.remove? - return c.remove(lowbits(v)) + return changed } // Max returns the highest value in the bitmap. @@ -337,7 +404,7 @@ func (b *Bitmap) Any() bool { // container should be removed from the bitmap though. for iter.Next() { _, c := iter.Value() - if c.n > 0 { + if c.N() > 0 { return true } } @@ -351,7 +418,6 @@ func (b *Bitmap) Size() int { for citer.Next() { _, c := citer.Value() numbytes += c.size() - } return numbytes } @@ -384,7 +450,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { continue } if k < ekey { - n += uint64(c.n) + n += uint64(c.N()) continue } if k == ekey { @@ -440,6 +506,8 @@ func (b *Bitmap) ForEachRange(start, end uint64, fn func(uint64)) { } // OffsetRange returns a new bitmap with a containers offset by start. +// The containers themselves are shared, so they get frozen so it will +// be safe to interact with them. func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { if lowbits(offset) != 0 { panic("offset must not contain low bits") @@ -454,13 +522,13 @@ func (b *Bitmap) OffsetRange(offset, start, end uint64) *Bitmap { off := highbits(offset) hi0, hi1 := highbits(start), highbits(end) citer, _ := b.Containers.Iterator(hi0) - other := NewBitmap() + other := NewSliceBitmap() for citer.Next() { k, c := citer.Value() if k >= hi1 { break } - other.Containers.Put(off+(k-hi0), c) + other.Containers.Put(off+(k-hi0), c.Freeze()) } return other } @@ -529,7 +597,10 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { b.unionIntoTargetSingle(output, others[0]) return output } - output := b.Clone() + // It may seem counterintuitive to freeze this, but the result is + // a new bitmap which can be safely modified, but postponing any + // allocations until an actual write to any given container. + output := b.Freeze() output.UnionInPlace(others...) return output } @@ -548,11 +619,11 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - target.Containers.Put(ki, ci.Clone()) + target.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - target.Containers.Put(kj, cj.Clone()) + target.Containers.Put(kj, cj.Freeze()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -685,11 +756,11 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { tContainer := target.Containers.Get(iKey) // if the target's full, short-circuit out. if tContainer != nil { - if tContainer.n == maxContainerVal+1 { + if tContainer.N() == maxContainerVal+1 { bitmapIters.markItersWithKeyAsHandled(i, iKey) continue } - expectedN = int64(tContainer.n) + expectedN = int64(tContainer.N()) } // Check i and later iters for any max-range containers, and // find out how many there are. @@ -699,7 +770,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // range that a container can store, so instead of calculating a // union we can generate an RLE container that represents the entire // range. - tContainer = NewContainerRun([]interval16{{start: 0, last: maxContainerVal}}) + tContainer = fullContainer target.Containers.Put(iKey, tContainer) bitmapIters.markItersWithKeyAsHandled(i, iKey) continue @@ -714,9 +785,9 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // No existing target container. if summaryStats.c == 1 { // There's no target and we have only one container, we - // can just clone it instead of unioning. + // can just reuse it instead of unioning. statsHit("unionInPlace/reuse") - target.Containers.Put(iKey, iContainer.Clone()) + target.Containers.Put(iKey, iContainer.Freeze()) bitmapIters[i].handled = true continue } @@ -726,17 +797,19 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // first other container, but for some cases, that will // result in cloning a non-bitmap, then converting it // to a bitmap, and this will be expensive... - if expectedN >= 512 && iContainer.typ != containerBitmap { + if expectedN >= 512 && iContainer.typ() != containerBitmap { // copying the non-bitmap, then converting it, // is expensive. statsHit("unionInPlace/newBitmap") - tContainer = NewContainerBitmap(0, nil) + tContainer = NewContainerBitmapN(nil, 0) itersToUnion = bitmapIters[i:] } else { // either N will be small or iContainer is a // bitmap, so we can skip one union op by copying it. + // And we can just freeze it, and the copy will + // happen later if it's needed... statsHit("unionInPlace/clone") - tContainer = iContainer.Clone() + tContainer = iContainer.Freeze() itersToUnion = bitmapIters[i+1:] } } else { @@ -745,13 +818,13 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // convert it preemptively, because union into a // bitmap is nearly always faster. itersToUnion = bitmapIters[i:] - if expectedN >= 512 && tContainer.typ != containerBitmap { + if expectedN >= 512 && tContainer.typ() != containerBitmap { statsHit("unionInPlace/convertToBitmap") - switch tContainer.typ { + switch tContainer.typ() { case containerArray: - tContainer.arrayToBitmap() + tContainer = tContainer.arrayToBitmap() case containerRun: - tContainer.runToBitmap() + tContainer = tContainer.runToBitmap() } } } @@ -762,6 +835,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { jKey, jContainer := iter.iter.Value() if iKey == jKey { + tContainer = tContainer.Thaw() tContainer.unionInPlace(jContainer) // "iter" is a local copy from the range // loop, not the actual slice member. @@ -798,7 +872,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { @@ -825,11 +899,11 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + output.Containers.Put(ki, ci.Freeze()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.Containers.Put(kj, cj.Clone()) + output.Containers.Put(kj, cj.Freeze()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj @@ -857,7 +931,7 @@ func (b *Bitmap) Shift(n int) (*Bitmap, error) { if lastCarry { o.add(0) } - if o.n > 0 { + if o.N() > 0 { output.Containers.Put(ki, o) } lastCarry = carry @@ -878,7 +952,7 @@ func (b *Bitmap) removeEmptyContainers() { citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() - if c.n == 0 { + if c.N() == 0 { b.Containers.Remove(k) } } @@ -888,7 +962,7 @@ func (b *Bitmap) countEmptyContainers() int { citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n == 0 { + if c.N() == 0 { result++ } } @@ -897,11 +971,9 @@ func (b *Bitmap) countEmptyContainers() int { // Optimize converts array and bitmap containers to run containers as necessary. func (b *Bitmap) Optimize() { - citer, _ := b.Containers.Iterator(0) - for citer.Next() { - _, c := citer.Value() - c.optimize() - } + b.Containers.UpdateEvery(func(c *Container, existed bool) (*Container, bool) { + return c.optimize(), true + }) } type errWriter struct { @@ -967,7 +1039,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { n: 0, } - ew.WriteUint32(byte4, cookie) + ew.WriteUint32(byte4, cookie|(uint32(b.Flags)<<24)) ew.WriteUint32(byte4, uint32(containerCount)) // Descriptive header section: encode keys and cardinality. @@ -979,10 +1051,10 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { // TODO: instead of commenting this out, we need to make it a configuration option //count := c.count() //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) - if c.n > 0 { + if c.N() > 0 { ew.WriteUint64(byte8, key) - ew.WriteUint16(byte2, uint16(c.typ)) - ew.WriteUint16(byte2, uint16(c.n-1)) + ew.WriteUint16(byte2, uint16(c.typ())) + ew.WriteUint16(byte2, uint16(c.N()-1)) } } @@ -993,7 +1065,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n > 0 { + if c.N() > 0 { ew.WriteUint32(byte4, offset) offset += uint32(c.size()) } @@ -1009,7 +1081,7 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { citer, _ = b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - if c.n > 0 { + if c.N() > 0 { nn, err := c.WriteTo(w) n += nn if err != nil { @@ -1029,7 +1101,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) + fileVersion := uint32(data[2]) + b.Flags = data[3] if fileMagic != MagicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } @@ -1038,8 +1111,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { return fmt.Errorf("wrong roaring version, file is v%d, server requires v%d", fileVersion, storageVersion) } - // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). - keyN := binary.LittleEndian.Uint32(data[4:8]) + // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)). + keyN := binary.LittleEndian.Uint32(data[3+1 : 8]) headerSize := headerBaseSize b.Containers.Reset() @@ -1065,13 +1138,17 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.typ { + // this shouldn't happen, since we don't normally store nils. + if c == nil { + continue + } + switch c.typ() { case containerRun: runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount]) opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n:c.n]) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) @@ -1219,6 +1296,11 @@ func (itr *Iterator) Seek(seek uint64) { return // eof } itr.key, itr.c = itr.citer.Value() + if roaringParanoia { + if itr.c == nil { + panic("seeking iterator got a nil container when Next() was true") + } + } // Move to the correct value index inside the container. lb := lowbits(seek) @@ -1274,7 +1356,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Iterate over containers until we find the next value or EOF. for { if itr.c.isArray() { - if itr.j >= itr.c.n-1 { + if itr.j >= itr.c.N()-1 { // Reached end of array, move to the next container. if !itr.citer.Next() { itr.c = nil @@ -1410,6 +1492,9 @@ func (c *Container) count() (n int32) { // countRange counts the number of bits set between [start, end). func (c *Container) countRange(start, end int32) (n int32) { + if c == nil { + return 0 + } if c.isArray() { return c.arrayCountRange(start, end) } else if c.isRun() { @@ -1494,73 +1579,78 @@ func (c *Container) runCountRange(start, end int32) (n int32) { } // add adds a value to the container. -func (c *Container) add(v uint16) (added bool) { - +func (c *Container) add(v uint16) (newC *Container, added bool) { + if c == nil { + return NewContainerArray([]uint16{v}), true + } if c.isArray() { - added = c.arrayAdd(v) + return c.arrayAdd(v) } else if c.isRun() { - added = c.runAdd(v) + return c.runAdd(v) } else { - added = c.bitmapAdd(v) + return c.bitmapAdd(v) } - if added { - c.n++ - } - return added } -func (c *Container) arrayAdd(v uint16) bool { +func (c *Container) arrayAdd(v uint16) (*Container, bool) { // Optimize appending to the end of an array container. array := c.array() - if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && array[c.n-1] < v { + if c.N() > 0 && c.N() < ArrayMaxSize && c.isArray() && array[c.N()-1] < v { statsHit("arrayAdd/append") - c.unmapArray() + c = c.Thaw() array = append(c.array(), v) c.setArray(array) - return true + return c, true } // Find index of the integer in the container. Exit if it already exists. i := search32(array, v) if i >= 0 { - return false + return c, false } // Convert to a bitmap container if too many values are in an array container. - if c.n >= ArrayMaxSize { + if c.N() >= ArrayMaxSize { statsHit("arrayAdd/arrayToBitmap") - c.arrayToBitmap() + c = c.arrayToBitmap() return c.bitmapAdd(v) } // Otherwise insert into array. statsHit("arrayAdd/insert") - c.unmapArray() + c = c.Thaw() i = -i - 1 array = append(c.array(), 0) copy(array[i+1:], array[i:]) array[i] = v c.setArray(array) - return true + return c, true } -func (c *Container) bitmapAdd(v uint16) bool { - if c.bitmapContains(v) { - return false +func (c *Container) bitmapAdd(v uint16) (*Container, bool) { + if c == nil { + c = NewContainerBitmapN(nil, 1) + c.bitmap()[v/64] |= (1 << uint64(v%64)) + return c, true } - c.unmapBitmap() + if c.bitmapContains(v) { + return c, false + } + c = c.Thaw() c.bitmap()[v/64] |= (1 << uint64(v%64)) - return true + c.setN(c.N() + 1) + return c, true } -func (c *Container) runAdd(v uint16) bool { +func (c *Container) runAdd(v uint16) (*Container, bool) { runs := c.runs() if len(runs) == 0 { - c.unmapRun() + c = c.Thaw() c.setRuns([]interval16{{start: v, last: v}}) - return true + c.setN(1) + return c, true } i := sort.Search(len(runs), @@ -1572,10 +1662,10 @@ func (c *Container) runAdd(v uint16) bool { iv := runs[i] if v >= iv.start && iv.last >= v { - return false + return c, false } - c.unmapRun() + c = c.Thaw() runs = c.runs() if iv.last < v { if iv.last == v-1 { @@ -1589,7 +1679,8 @@ func (c *Container) runAdd(v uint16) bool { runs[i-1].last = iv.last runs = append(runs[:i], runs[i+1:]...) c.setRuns(runs) - return true + c.setN(c.N() + 1) + return c, true } // just before an interval runs[i].start-- @@ -1602,11 +1693,15 @@ func (c *Container) runAdd(v uint16) bool { runs = append(runs[:i], append([]interval16{newIv}, runs[i:]...)...) } c.setRuns(runs) - return true + c.setN(c.N() + 1) + return c, true } // Contains returns true if v is in the container. func (c *Container) Contains(v uint16) bool { + if c == nil { + return false + } if c.isArray() { return c.arrayContains(v) } else if c.isRun() { @@ -1660,17 +1755,17 @@ func (c *Container) countRuns() (r int32) { // optimize converts the container to the type which will take up the least // amount of space. -func (c *Container) optimize() { - if c.n == 0 { +func (c *Container) optimize() *Container { + if c.N() == 0 { statsHit("optimize/empty") - return + return nil } runs := c.countRuns() var newType byte - if runs <= runMaxSize && runs <= c.n/2 { + if runs <= runMaxSize && runs <= c.N()/2 { newType = containerRun - } else if c.n < ArrayMaxSize { + } else if c.N() < ArrayMaxSize { newType = containerArray } else { newType = containerBitmap @@ -1680,75 +1775,94 @@ func (c *Container) optimize() { if c.isArray() { if newType == containerBitmap { statsHit("optimize/arrayToBitmap") - c.arrayToBitmap() + c = c.arrayToBitmap() } else if newType == containerRun { statsHit("optimize/arrayToRun") - c.arrayToRun(runs) + c = c.arrayToRun(runs) } else { statsHit("optimize/arrayUnchanged") } } else if c.isBitmap() { if newType == containerArray { statsHit("optimize/bitmapToArray") - c.bitmapToArray() + c = c.bitmapToArray() } else if newType == containerRun { statsHit("optimize/bitmapToRun") - c.bitmapToRun(runs) + c = c.bitmapToRun(runs) } else { statsHit("optimize/bitmapUnchanged") } } else if c.isRun() { if newType == containerBitmap { statsHit("optimize/runToBitmap") - c.runToBitmap() + c = c.runToBitmap() } else if newType == containerArray { statsHit("optimize/runToArray") - c.runToArray() + c = c.runToArray() } else { statsHit("optimize/runUnchanged") } } + return c } // unionInPlace does not necessarily preserve container's N; it's expected // to be used when running a sequence of unions, after which you should // call Repair(). (As of this writing, that only matters for bitmaps.) -func (c *Container) unionInPlace(other *Container) { - switch c.typ { +// +// If called on a frozen container, or a container of the wrong sort, +// it is possible that the returned container will not actually be the +// original container; in-place is a suggestion. +func (c *Container) unionInPlace(other *Container) *Container { + if c == nil { + return other.Freeze() + } + if other == nil { + return c + } + // short-circuit the trivial cases + if c.N() == maxContainerVal+1 || other.N() == maxContainerVal+1 { + return fullContainer + } + switch c.typ() { case containerBitmap: - switch other.typ { + switch other.typ() { case containerBitmap: - unionBitmapBitmapInPlace(c, other) + return unionBitmapBitmapInPlace(c, other) case containerArray: - unionBitmapArrayInPlace(c, other) + return unionBitmapArrayInPlace(c, other) case containerRun: - unionBitmapRunInPlace(c, other) + return unionBitmapRunInPlace(c, other) } case containerArray: - switch other.typ { + switch other.typ() { case containerBitmap: - c.arrayToBitmap() - unionBitmapBitmapInPlace(c, other) + c = c.arrayToBitmap() + return unionBitmapBitmapInPlace(c, other) case containerArray: - unionArrayArrayInPlace(c, other) + return unionArrayArrayInPlace(c, other) case containerRun: - c.arrayToBitmap() - unionBitmapRunInPlace(c, other) + c = c.arrayToBitmap() + return unionBitmapRunInPlace(c, other) } case containerRun: - switch other.typ { + switch other.typ() { case containerBitmap: - c.runToBitmap() - unionBitmapBitmapInPlace(c, other) + c = c.runToBitmap() + return unionBitmapBitmapInPlace(c, other) case containerArray: - c.runToBitmap() - unionBitmapArrayInPlace(c, other) + c = c.runToBitmap() + return unionBitmapArrayInPlace(c, other) case containerRun: - c.runToBitmap() - unionBitmapRunInPlace(c, other) + c = c.runToBitmap() + return unionBitmapRunInPlace(c, other) } } + if roaringParanoia { + panic(fmt.Sprintf("invalid union op: unknown types %d/%d", c.typ(), other.typ())) + } + return c } func (c *Container) arrayContains(v uint16) bool { @@ -1779,58 +1893,71 @@ func (c *Container) runContains(v uint16) bool { } // remove removes a value from the container. -func (c *Container) remove(v uint16) (removed bool) { - if c.isArray() { - removed = c.arrayRemove(v) - } else if c.isRun() { - removed = c.runRemove(v) - } else { - removed = c.bitmapRemove(v) +func (c *Container) remove(v uint16) (newC *Container, removed bool) { + if c == nil { + return nil, false + } + if c.isArray() { + return c.arrayRemove(v) + } else if c.isRun() { + return c.runRemove(v) + } else { + return c.bitmapRemove(v) } - return removed } -func (c *Container) arrayRemove(v uint16) bool { +func (c *Container) arrayRemove(v uint16) (*Container, bool) { array := c.array() i := search32(array, v) if i < 0 { - return false + return c, false } - c.unmapArray() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() array = c.array() array = append(array[:i], array[i+1:]...) - c.n-- c.setArray(array) - return true + return c, true } -func (c *Container) bitmapRemove(v uint16) bool { +func (c *Container) bitmapRemove(v uint16) (*Container, bool) { if !c.bitmapContains(v) { - return false + return c, false } - c.unmapBitmap() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() // Lower count and remove element. c.bitmap()[v/64] &^= (uint64(1) << uint(v%64)) - c.n-- + c.setN(c.N() - 1) // Convert to array if we go below the threshold. - if c.n == ArrayMaxSize { + if c.N() == ArrayMaxSize { statsHit("bitmapRemove/bitmapToArray") - c.bitmapToArray() + c = c.bitmapToArray() } - return true + return c, true } // runRemove removes v from a run container, and returns true if v was removed. -func (c *Container) runRemove(v uint16) bool { +func (c *Container) runRemove(v uint16) (*Container, bool) { runs := c.runs() i, contains := binSearchRuns(v, runs) if !contains { - return false + return c, false } - c.unmapRun() + // removing the last item? we can just return the empty container. + if c.N() == 1 { + return nil, true + } + c = c.Thaw() runs = c.runs() if v == runs[i].last && v == runs[i].start { runs = append(runs[:i], runs[i+1:]...) @@ -1846,13 +1973,17 @@ func (c *Container) runRemove(v uint16) bool { runs[i+1] = interval16{start: v + 1, last: last} // runs = append(runs[:i+1], append([]interval16{{start: v + 1, last: last}}, runs[i+1:]...)...) } - c.n-- + c.setN(c.N() - 1) c.setRuns(runs) - return true + return c, true } // max returns the maximum value in the container. func (c *Container) max() uint16 { + if c == nil || c.N() == 0 { + // probably wrong, but prevents a crash elsewhere + return 0 + } if c.isArray() { return c.arrayMax() } else if c.isRun() { @@ -1864,9 +1995,6 @@ func (c *Container) max() uint16 { func (c *Container) arrayMax() uint16 { array := c.array() - if len(array) == 0 { - return 0 // probably hiding some ugly bug but it prevents a crash - } return array[len(array)-1] } @@ -1894,26 +2022,33 @@ func (c *Container) runMax() uint16 { } // bitmapToArray converts from bitmap format to array format. -func (c *Container) bitmapToArray() { +func (c *Container) bitmapToArray() *Container { statsHit("bitmapToArray") - bitmap := c.bitmap() - c.setBitmap(nil) - c.typ = containerArray - c.mapped = false - - // return early if empty - if c.n == 0 { - c.setArray(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for bitmapToArray") + } + return nil } + // If c is frozen, we'll be making a new array container. Otherwise, + // we'll convert this container. + if c.N() == 0 { + if c.frozen() { + return NewContainerArray(nil) + } + c.setTyp(containerArray) + c.setArray(nil) + return c + } + bitmap := c.bitmap() n := int32(0) - array := make([]uint16, c.n) + array := make([]uint16, c.N()) for i, word := range bitmap { for word != 0 { t := word & -word if roaringParanoia { - if n >= c.n { + if n >= c.N() { panic("bitmap has more bits set than container.n") } } @@ -1923,71 +2058,112 @@ func (c *Container) bitmapToArray() { } } if roaringParanoia { - if n != c.n { + if n != c.N() { panic("bitmap has fewer bits set than container.n") } } + if c.frozen() { + return NewContainerArray(array) + } + c.setTyp(containerArray) + c.setMapped(false) c.setArray(array) + return c } // arrayToBitmap converts from array format to bitmap format. -func (c *Container) arrayToBitmap() { +func (c *Container) arrayToBitmap() *Container { statsHit("arrayToBitmap") - array := c.array() - c.typ = containerBitmap - bitmap := make([]uint64, bitmapN) - c.setBitmap(bitmap) - c.mapped = false + if c == nil { + if roaringParanoia { + panic("nil container for arrayToBitmap") + } + return nil + } // return early if empty - if c.n == 0 { - return + if c.N() == 0 { + if c.frozen() { + return NewContainerBitmap(0, nil) + } + c.setTyp(containerBitmap) + c.setBitmap(make([]uint64, bitmapN)) + return c } - for _, v := range array { + bitmap := make([]uint64, bitmapN) + for _, v := range c.array() { bitmap[int(v)/64] |= (uint64(1) << uint(v%64)) } + if c.frozen() { + return NewContainerBitmapN(bitmap, c.N()) + } + c.setTyp(containerBitmap) + c.setMapped(false) + c.setBitmap(bitmap) + return c } // runToBitmap converts from RLE format to bitmap format. -func (c *Container) runToBitmap() { +func (c *Container) runToBitmap() *Container { statsHit("runToBitmap") - runs := c.runs() - bitmap := make([]uint64, bitmapN) - c.typ = containerBitmap - c.setBitmap(bitmap) - - c.mapped = false - - // return early if empty - if c.n == 0 { - return + if c == nil { + if roaringParanoia { + panic("nil container for runToBitmap") + } + return nil } - for _, r := range runs { + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerBitmap(0, nil) + } + c.setTyp(containerBitmap) + c.setBitmap(make([]uint64, bitmapN)) + return c + } + bitmap := make([]uint64, bitmapN) + for _, r := range c.runs() { // TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits //note v must be int or will overflow for v := int(r.start); v <= int(r.last); v++ { bitmap[v/64] |= (uint64(1) << uint(v%64)) } } + if c.frozen() { + return NewContainerBitmapN(bitmap, c.N()) + } + c.setTyp(containerBitmap) + c.setMapped(false) + c.setBitmap(bitmap) + return c } // bitmapToRun converts from bitmap format to RLE format. -func (c *Container) bitmapToRun(numRuns int32) { +func (c *Container) bitmapToRun(numRuns int32) *Container { statsHit("bitmapToRun") - bitmap := c.bitmap() - c.mapped = false - c.typ = containerRun - // return early if empty - if c.n == 0 { - c.setRuns(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for bitmapToRun") + } + return nil } + + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerRun(nil) + } + c.setTyp(containerRun) + c.setRuns(nil) + return c + } + + bitmap := c.bitmap() if numRuns == 0 { numRuns = bitmapCountRuns(bitmap) } - runs := make([]interval16, 0, numRuns) current := bitmap[0] @@ -2027,20 +2203,37 @@ func (c *Container) bitmapToRun(numRuns int32) { // pad LSBs with 0s current = current & (current + 1) } + if c.frozen() { + return NewContainerRunN(runs, c.N()) + } + c.setTyp(containerRun) c.setRuns(runs) + c.setMapped(false) + return c } // arrayToRun converts from array format to RLE format. -func (c *Container) arrayToRun(numRuns int32) { +func (c *Container) arrayToRun(numRuns int32) *Container { statsHit("arrayToRun") - array := c.array() - c.typ = containerRun - c.mapped = false - // return early if empty - if c.n == 0 { - c.setRuns(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for arrayToRun") + } + return nil } + + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerRun(nil) + } + c.setTyp(containerRun) + c.setRuns(nil) + return c + } + + array := c.array() + if numRuns == 0 { numRuns = arrayCountRuns(array) } @@ -2055,24 +2248,39 @@ func (c *Container) arrayToRun(numRuns int32) { } } // append final run - runs = append(runs, interval16{start, array[c.n-1]}) + runs = append(runs, interval16{start, array[c.N()-1]}) + if c.frozen() { + return NewContainerRunN(runs, c.N()) + } + c.setTyp(containerRun) + c.setMapped(false) c.setRuns(runs) + return c } // runToArray converts from RLE format to array format. -func (c *Container) runToArray() { +func (c *Container) runToArray() *Container { statsHit("runToArray") - runs := c.runs() - c.typ = containerArray - c.mapped = false - - // return early if empty - if c.n == 0 { - c.setArray(nil) - return + if c == nil { + if roaringParanoia { + panic("nil container for runToArray") + } + return nil } - array := make([]uint16, c.n) + // return early if empty + if c.N() == 0 { + if c.frozen() { + return NewContainerArray(nil) + } + c.setTyp(containerArray) + c.setArray(nil) + return c + } + + runs := c.runs() + + array := make([]uint16, c.N()) n := int32(0) for _, r := range runs { for v := int(r.start); v <= int(r.last); v++ { @@ -2081,38 +2289,43 @@ func (c *Container) runToArray() { } } if roaringParanoia { - if n != c.n { + if n != c.N() { panic("run has fewer bits set than container.n") } } + if c.frozen() { + return NewContainerArray(array) + } + c.setTyp(containerArray) + c.setMapped(false) c.setArray(array) + return c } // Clone returns a copy of c. func (c *Container) Clone() (out *Container) { statsHit("Container/Clone") - switch c.typ { + if c == nil { + return nil + } + switch c.typ() { case containerArray: statsHit("Container/Clone/Array") - cArray := c.array() - array := make([]uint16, len(cArray)) - copy(array, cArray) - out = NewContainerArray(array) + out = NewContainerArrayCopy(c.array()) case containerBitmap: statsHit("Container/Clone/Bitmap") - other := NewContainerBitmap(c.n, nil) + other := NewContainerBitmapN(nil, c.N()) copy(other.bitmap(), c.bitmap()) out = other case containerRun: statsHit("Container/Clone/Run") - cRuns := c.runs() - runs := make([]interval16, len(cRuns)) - copy(runs, cRuns) - out = NewContainerRun(runs) + out = NewContainerRunCopy(c.runs()) + default: + panic(fmt.Sprintf("cloning a container of unknown type %d", c.typ())) } // this should probably never happen if roaringParanoia { - if out.n != out.count() { + if out.N() != out.count() { panic("cloned container has wrong n") } } @@ -2121,6 +2334,9 @@ func (c *Container) Clone() (out *Container) { // WriteTo writes c to w. func (c *Container) WriteTo(w io.Writer) (n int64, err error) { + if c == nil { + return 0, nil + } if c.isArray() { return c.arrayWriteTo(w) } else if c.isRun() { @@ -2144,7 +2360,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { //} // Write sizeof(uint16) * cardinality bytes. - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&array[0]))[: 2*c.n : 2*c.n]) + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&array[0]))[: 2*c.N() : 2*c.N()]) return int64(nn), err } @@ -2185,7 +2401,12 @@ func (c *Container) size() int { // info returns the current stats about the container. func (c *Container) info() containerInfo { - info := containerInfo{N: c.n} + info := containerInfo{N: c.N()} + if c == nil { + info.Type = "nil" + info.Alloc = 0 + return info + } if c.isArray() { info.Type = "array" @@ -2198,7 +2419,7 @@ func (c *Container) info() containerInfo { info.Alloc = len(c.bitmap()) * 8 // sizeof(uint64) } - if c.mapped { + if c.Mapped() { if c.isArray() { info.Pointer = unsafe.Pointer(&c.array()[0]) } else if c.isRun() { @@ -2215,24 +2436,27 @@ func (c *Container) info() containerInfo { func (c *Container) check() error { var a ErrorList + if c == nil { + return nil + } if c.isArray() { array := c.array() - if int32(len(array)) != c.n { - a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.n)) + if int32(len(array)) != c.N() { + a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N())) } } else if c.isRun() { n := c.runCountRange(0, maxContainerVal+1) - if n != c.n { - a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.n)) + if n != c.N() { + a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N())) } } else if c.isBitmap() { - if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.n { - a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.n)) + if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.N() { + a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N())) } } else { a.Append(fmt.Errorf("empty container")) - if c.n != 0 { - a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.n)) + if c.N() != 0 { + a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.N())) } } @@ -2245,6 +2469,11 @@ func (c *Container) check() error { // Repair repairs the cardinality of c if it has been corrupted by // optimized operations. func (c *Container) Repair() { + // a frozen container can't have had n or contents changed, so we + // don't need to recount it. + if c.frozen() { + return + } if c.isBitmap() { c.bitmapRepair() } @@ -2262,7 +2491,7 @@ func (c *Container) bitmapRepair() { n += int32(popcount(bitmap[i+2])) n += int32(popcount(bitmap[i+3])) } - c.n = n + c.setN(n) } // containerInfo represents a point-in-time snapshot of container stats. @@ -2290,20 +2519,20 @@ func flipArray(b *Container) *Container { statsHit("flipArray") // TODO: actually implement this x := b.Clone() - x.arrayToBitmap() + x = x.arrayToBitmap() return flipBitmap(x) } func flipBitmap(b *Container) *Container { statsHit("flipBitmap") - other := NewContainerBitmap(0, nil) + other := NewContainerBitmapN(nil, 0) bitmap := b.bitmap() otherBitmap := other.bitmap() for i, word := range bitmap { otherBitmap[i] = ^word } - other.n = other.count() + other.setN(other.count()) return other } @@ -2311,11 +2540,20 @@ func flipRun(b *Container) *Container { statsHit("flipRun") // TODO: actually implement this x := b.Clone() - x.runToBitmap() + x = x.runToBitmap() return flipBitmap(x) } func intersectionCount(a, b *Container) int32 { + if a.N() == maxContainerVal+1 { + return b.N() + } + if b.N() == maxContainerVal+1 { + return a.N() + } + if a.N() == 0 || b.N() == 0 { + return 0 + } if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) @@ -2347,9 +2585,6 @@ func intersectionCountArrayArray(a, b *Container) (n int32) { statsHit("intersectionCount/ArrayArray") ca, cb := a.array(), b.array() na, nb := len(ca), len(cb) - if na == 0 || nb == 0 { - return 0 - } if na > nb { ca, cb = cb, ca na, nb = nb, na // nolint: ineffassign @@ -2449,6 +2684,15 @@ func intersectionCountBitmapBitmap(a, b *Container) (n int32) { } func intersect(a, b *Container) *Container { + if a.N() == maxContainerVal+1 { + return b.Freeze() + } + if b.N() == maxContainerVal+1 { + return a.Freeze() + } + if a.N() == 0 || b.N() == 0 { + return nil + } if a.isArray() { if b.isArray() { return intersectArrayArray(a, b) @@ -2523,6 +2767,7 @@ func intersectRunRun(a, b *Container) *Container { output := NewContainerRun(nil) ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) + n := int32(0) for i, j := 0, 0; i < na && j < nb; { va, vb := ra[i], rb[j] if va.last < vb.start { @@ -2533,24 +2778,25 @@ func intersectRunRun(a, b *Container) *Container { j++ } else if va.last > vb.last && va.start >= vb.start { // |--vb-|-|-va--| - output.n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) + n += output.runAppendInterval(interval16{start: va.start, last: vb.last}) j++ } else if va.last > vb.last && va.start < vb.start { // |--va|--vb--|--| - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else if va.last <= vb.last && va.start >= vb.start { // |--vb|--va--|--| - output.n += output.runAppendInterval(va) + n += output.runAppendInterval(va) i++ } else if va.last <= vb.last && va.start < vb.start { // |--va-|-|-vb--| - output.n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) + n += output.runAppendInterval(interval16{start: vb.start, last: va.last}) i++ } } + output.setN(n) runs := output.runs() - if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { + if n < ArrayMaxSize && int32(len(runs)) > n/2 { output.runToArray() } else if len(runs) > runMaxSize { output.runToBitmap() @@ -2564,9 +2810,9 @@ func intersectBitmapRun(a, b *Container) *Container { statsHit("intersect/BitmapRun") var output *Container runs := b.runs() - if b.n <= ArrayMaxSize || a.n <= ArrayMaxSize { + if b.N() <= ArrayMaxSize || a.N() <= ArrayMaxSize { // output is array container - array := make([]uint16, 0, b.n) + array := make([]uint16, 0, b.N()) for _, iv := range runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { @@ -2587,6 +2833,7 @@ func intersectBitmapRun(a, b *Container) *Container { output = NewContainerBitmap(0, nil) bitmap := output.bitmap() aBitmap := a.bitmap() + n := int32(0) for j := 0; j < len(runs); j++ { vb := runs[j] i := vb.start >> 6 // index into a @@ -2595,22 +2842,22 @@ func intersectBitmapRun(a, b *Container) *Container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b bitmap[i] = aBitmap[i] - output.n += int32(popcount(aBitmap[i])) + n += int32(popcount(aBitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := aBitmap[i] & mask bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (aBitmap[i] >> offset) << offset bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (aBitmap[i] << offset) >> offset bitmap[i] |= bits - output.n += int32(popcount(bits)) + n += int32(popcount(bits)) } // update loop vars i++ @@ -2618,6 +2865,7 @@ func intersectBitmapRun(a, b *Container) *Container { valast = vastart + 63 } } + output.setN(n) } return output } @@ -2653,11 +2901,14 @@ func intersectBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) + output := NewContainerBitmapN(ob, n) return output } func union(a, b *Container) *Container { + if a.N() == maxContainerVal+1 || b.N() == maxContainerVal+1 { + return fullContainer + } if a.isArray() { if b.isArray() { return unionArrayArray(a, b) @@ -2727,7 +2978,7 @@ func unionArrayArray(a, b *Container) *Container { // unionArrayArrayInPlace does what it sounds like -- tries to combine // the two arrays in-place. It does not try to ensure that the result is // of a good array size, so it could be up to twice that size, temporarily. -func unionArrayArrayInPlace(a, b *Container) { +func unionArrayArrayInPlace(a, b *Container) *Container { statsHit("union/ArrayArrayInPlace") aa, ab := a.array(), b.array() na, nb := len(aa), len(ab) @@ -2762,25 +3013,26 @@ func unionArrayArrayInPlace(a, b *Container) { j++ } } - a.setArray(output[:outN]) - a.n = int32(outN) - if a.n > ArrayMaxSize { - a.optimize() + // a union can't omit anything that was previously in a, so if + // the output is the same length, nothing changed. + if len(output) != int(a.N()) { + a = a.Thaw() + a.setArray(output[:outN]) + a = a.optimize() } + return a } // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *Container) *Container { statsHit("union/ArrayRun") - if b.n == maxContainerVal+1 { - return b.Clone() - } output := NewContainerRun(nil) aa, rb := a.array(), b.runs() na, nb := len(aa), len(rb) var vb interval16 var va uint16 + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na { va = aa[i] @@ -2789,17 +3041,18 @@ func unionArrayRun(a, b *Container) *Container { vb = rb[j] } if i < na && (j >= nb || va < vb.start) { - output.n += output.runAppendInterval(interval16{start: va, last: va}) + n += output.runAppendInterval(interval16{start: va, last: va}) i++ } else { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } } - if output.n < ArrayMaxSize { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.runToArray() } else if len(output.runs()) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -2836,16 +3089,11 @@ func (c *Container) runAppendInterval(v interval16) int32 { func unionRunRun(a, b *Container) *Container { statsHit("union/RunRun") - if a.n == maxContainerVal+1 { - return a.Clone() - } - if b.n == maxContainerVal+1 { - return b.Clone() - } ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) output := NewContainerRun(make([]interval16, 0, na+nb)) var va, vb interval16 + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na { va = ra[i] @@ -2854,13 +3102,14 @@ func unionRunRun(a, b *Container) *Container { vb = rb[j] } if i < na && (j >= nb || va.start < vb.start) { - output.n += output.runAppendInterval(va) + n += output.runAppendInterval(va) i++ } else { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } } + output.setN(n) if len(output.runs()) > runMaxSize { output.runToBitmap() } @@ -2869,55 +3118,52 @@ func unionRunRun(a, b *Container) *Container { func unionBitmapRun(a, b *Container) *Container { statsHit("union/BitmapRun") - if b.n == maxContainerVal+1 { - return b.Clone() - } - if a.n == maxContainerVal+1 { - return a.Clone() - } output := a.Clone() - bitmap := output.bitmap() for _, run := range b.runs() { - output.bitmapSetRange(bitmap, uint64(run.start), uint64(run.last)+1) + output.bitmapSetRange(uint64(run.start), uint64(run.last)+1) } return output } // unions the run b into the bitmap a, mutating a in place. The n value of // a will need to be repaired after the fact. -func unionBitmapRunInPlace(a, b *Container) { - a.unmapBitmap() +func unionBitmapRunInPlace(a, b *Container) *Container { + a = a.Thaw() bitmap := a.bitmap() statsHit("union/BitmapRun") for _, run := range b.runs() { bitmapSetRangeIgnoreN(bitmap, uint64(run.start), uint64(run.last)+1) } + return a } const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j) (c must be a bitmap container, and bitmap must // be its bitmap). -func (c *Container) bitmapSetRange(bitmap []uint64, i, j uint64) { +func (c *Container) bitmapSetRange(i, j uint64) { + bitmap := c.bitmap() x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcount(X) ycnt := popcount(Y) + n := int32(c.N()) if x == y { - c.n += int32((j - i) - popcount(bitmap[x]&(X&Y))) + n += int32((j - i) - popcount(bitmap[x]&(X&Y))) bitmap[x] |= (X & Y) } else { - c.n += int32(xcnt - popcount(bitmap[x]&X)) + n += int32(xcnt - popcount(bitmap[x]&X)) bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int32(64 - popcount(bitmap[i])) + n += int32(64 - popcount(bitmap[i])) bitmap[i] = maxBitmap } - c.n += int32(ycnt - popcount(bitmap[y]&Y)) + n += int32(ycnt - popcount(bitmap[y]&Y)) bitmap[y] |= Y } + c.setN(n) } // sets all bits in [i, j) without updating any corresponding n value. @@ -2945,23 +3191,25 @@ func (c *Container) bitmapXorRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) bitmap := c.bitmap() + n := c.N() if x == y { cnt := popcount(bitmap[x]) bitmap[x] ^= (X & Y) //// flip - c.n += int32(popcount(bitmap[x]) - cnt) + n += int32(popcount(bitmap[x]) - cnt) } else { cnt := popcount(bitmap[x]) bitmap[x] ^= X - c.n += int32(popcount(bitmap[x]) - cnt) + n += int32(popcount(bitmap[x]) - cnt) for i := x + 1; i < y; i++ { cnt = popcount(bitmap[i]) bitmap[i] ^= maxBitmap - c.n += int32(popcount(bitmap[i]) - cnt) + n += int32(popcount(bitmap[i]) - cnt) } cnt = popcount(bitmap[y]) bitmap[y] ^= Y - c.n += int32(popcount(bitmap[y]) - cnt) + n += int32(popcount(bitmap[y]) - cnt) } + c.setN(n) } // zeroes all bits in [i, j) (c must be a bitmap container) @@ -2971,26 +3219,34 @@ func (c *Container) bitmapZeroRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) bitmap := c.bitmap() + n := c.N() if x == y { - c.n -= int32(popcount(bitmap[x] & (X & Y))) + n -= int32(popcount(bitmap[x] & (X & Y))) bitmap[x] &= ^(X & Y) } else { - c.n -= int32(popcount(bitmap[x] & X)) + n -= int32(popcount(bitmap[x] & X)) bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int32(popcount(bitmap[i])) + n -= int32(popcount(bitmap[i])) bitmap[i] = 0 } - c.n -= int32(popcount(bitmap[y] & Y)) + n -= int32(popcount(bitmap[y] & Y)) bitmap[y] &= ^Y } + c.setN(n) } +// equals reports whether two containers are equal. func (c *Container) equals(c2 *Container) bool { - if c.mapped != c2.mapped || c.typ != c2.typ || c.n != c2.n { + if c == nil || c2 == nil { + if c != c2 { + return false + } + } + if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() { return false } - if c.typ == containerArray { + if c.typ() == containerArray { ca, c2a := c.array(), c2.array() if len(ca) != len(c2a) { return false @@ -3000,7 +3256,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.typ == containerBitmap { + } else if c.typ() == containerBitmap { cb, c2b := c.bitmap(), c2.bitmap() if len(cb) != len(c2b) { return false @@ -3010,7 +3266,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.typ == containerRun { + } else if c.typ() == containerRun { cr, c2r := c.runs(), c2.runs() if len(cr) != len(c2r) { return false @@ -3021,7 +3277,7 @@ func (c *Container) equals(c2 *Container) bool { } } } else { - panic(fmt.Sprintf("unknown container type: %v", c.typ)) + panic(fmt.Sprintf("unknown container type: %v", c.typ())) } return true } @@ -3029,23 +3285,26 @@ func (c *Container) equals(c2 *Container) bool { func unionArrayBitmap(a, b *Container) *Container { output := b.Clone() bitmap := output.bitmap() + n := output.N() for _, v := range a.array() { if !output.bitmapContains(v) { bitmap[v/64] |= (1 << uint64(v%64)) - output.n++ + n++ } } + output.setN(n) return output } // unions array b into bitmap a, mutating a in place. The n value // of a will need to be repaired after the fact. -func unionBitmapArrayInPlace(a, b *Container) { - a.unmapBitmap() +func unionBitmapArrayInPlace(a, b *Container) *Container { + a = a.Thaw() bitmap := a.bitmap() for _, v := range b.array() { bitmap[v>>6] |= (uint64(1) << (v % 64)) } + return a } func unionBitmapBitmap(a, b *Container) *Container { @@ -3065,15 +3324,14 @@ func unionBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) + output := NewContainerBitmapN(ob, n) return output } // unions bitmap b into bitmap a, mutating a in place. The n value of // a will need to be repaired after the fact. -func unionBitmapBitmapInPlace(a, b *Container) { - - a.unmapBitmap() +func unionBitmapBitmapInPlace(a, b *Container) *Container { + a = a.Thaw() // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html @@ -3090,9 +3348,16 @@ func unionBitmapBitmapInPlace(a, b *Container) { ab[i+2] |= bb[i+2] ab[i+3] |= bb[i+3] } + return a } func difference(a, b *Container) *Container { + if a.N() == 0 || b.N() == maxContainerVal+1 { + return nil + } + if b.N() == 0 { + return a.Freeze() + } if a.isArray() { if b.isArray() { return differenceArrayArray(a, b) @@ -3151,12 +3416,7 @@ func differenceArrayArray(a, b *Container) *Container { func differenceArrayRun(a, b *Container) *Container { statsHit("difference/ArrayRun") // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - - if a.n == 0 || b.n == 0 { - return a.Clone() - } - - output := NewContainerArray(make([]uint16, 0, a.n)) + output := make([]uint16, 0, a.N()) // cardinality upper bound: card(A) i := 0 // array index @@ -3164,11 +3424,11 @@ func differenceArrayRun(a, b *Container) *Container { aa, rb := a.array(), b.runs() // handle overlap - for i < int(a.n) { + for i < len(aa) { // keep all array elements before beginning of runs if aa[i] < rb[j].start { - output.add(aa[i]) + output = append(output, aa[i]) i++ continue } @@ -3192,29 +3452,14 @@ func differenceArrayRun(a, b *Container) *Container { // keep all array elements after end of runs // It's possible that output was converted from array to bitmap in output.add() // so check container type before proceeding. - if output.typ == containerArray { - array := output.array() - array = append(array, aa[i:]...) - output.setArray(array) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += int32(len(aa[i:])) - } else { - for _, v := range aa[i:] { - output.add(v) - } - } + output = append(output, aa[i:]...) } - return output + return NewContainerArray(output) } // differenceBitmapRun computes the difference of an bitmap from a run. func differenceBitmapRun(a, b *Container) *Container { statsHit("difference/BitmapRun") - if a.n == 0 || b.n == 0 { - return a.Clone() - } - output := a.Clone() for _, run := range b.runs() { output.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) @@ -3226,9 +3471,6 @@ func differenceBitmapRun(a, b *Container) *Container { // container. func differenceRunArray(a, b *Container) *Container { statsHit("difference/RunArray") - if a.n == 0 || b.n == 0 { - return a.Clone() - } ra, ab := a.runs(), b.array() runs := make([]interval16, 0, len(ra)) @@ -3275,7 +3517,7 @@ RUNLOOP: } } output := NewContainerRun(runs) - output.optimize() + output = output.optimize() return output } @@ -3287,18 +3529,12 @@ func differenceRunBitmap(a, b *Container) *Container { if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { return flipBitmap(b) } - output := NewContainerRun(nil) - runs := output.runs() - if len(ra) == 0 { - return NewContainerRun(nil) - } - output.n = a.n + runs := make([]interval16, 0, len(ra)) for _, inputRun := range ra { run := inputRun add := true for bit := inputRun.start; bit <= inputRun.last; bit++ { if b.bitmapContains(bit) { - output.n-- if run.start == bit { if bit == 65535 { //overflow add = false @@ -3331,11 +3567,11 @@ func differenceRunBitmap(a, b *Container) *Container { } } - output.setRuns(runs) - if output.n < ArrayMaxSize && int32(len(runs)) > output.n/2 { - output.runToArray() + output := NewContainerRun(runs) + if output.N() < ArrayMaxSize && int32(len(runs)) > output.N()/2 { + output = output.runToArray() } else if len(runs) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -3343,9 +3579,6 @@ func differenceRunBitmap(a, b *Container) *Container { // differenceRunRun computes the difference of two runs. func differenceRunRun(a, b *Container) *Container { statsHit("difference/RunRun") - if a.n == 0 || b.n == 0 { - return a.Clone() - } ra, rb := a.runs(), b.runs() apos := 0 // current a-run index @@ -3406,7 +3639,7 @@ func differenceRunRun(a, b *Container) *Container { func differenceArrayBitmap(a, b *Container) *Container { statsHit("difference/ArrayBitmap") - output := make([]uint16, 0, a.n) + output := make([]uint16, 0, a.N()) bitmap := b.bitmap() for _, va := range a.array() { bmidx := va / 64 @@ -3426,14 +3659,16 @@ func differenceBitmapArray(a, b *Container) *Container { output := a.Clone() bitmap := output.bitmap() + n := output.N() for _, v := range b.array() { if output.bitmapContains(v) { bitmap[v/64] &^= (uint64(1) << uint(v%64)) - output.n-- + n-- } } - if output.n < ArrayMaxSize { - output.bitmapToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.bitmapToArray() } return output } @@ -3456,14 +3691,20 @@ func differenceBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) - if output.n < ArrayMaxSize { - output.bitmapToArray() + output := NewContainerBitmapN(ob, n) + if output.N() < ArrayMaxSize { + output = output.bitmapToArray() } return output } func xor(a, b *Container) *Container { + if a.N() == 0 { + return b.Freeze() + } + if b.N() == 0 { + return a.Freeze() + } if a.isArray() { if b.isArray() { return xorArrayArray(a, b) @@ -3493,33 +3734,33 @@ func xor(a, b *Container) *Container { func xorArrayArray(a, b *Container) *Container { statsHit("xor/ArrayArray") - output := NewContainerArray(nil) + output := make([]uint16, 0) aa, ab := a.array(), b.array() na, nb := len(aa), len(ab) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { - output.add(aa[i]) + output = append(output, aa[i]) i++ continue } else if i >= na && j < nb { - output.add(ab[j]) + output = append(output, ab[j]) j++ continue } va, vb := aa[i], ab[j] if va < vb { - output.add(va) + output = append(output, va) i++ } else if va > vb { - output.add(vb) + output = append(output, vb) j++ } else { //== i++ j++ } } - return output + return NewContainerArray(output) } func xorArrayBitmap(a, b *Container) *Container { @@ -3527,16 +3768,16 @@ func xorArrayBitmap(a, b *Container) *Container { output := b.Clone() for _, v := range a.array() { if b.bitmapContains(v) { - output.remove(v) + output, _ = output.remove(v) } else { - output.add(v) + output, _ = output.add(v) } } // It's possible that output was converted from bitmap to array in output.remove() // so we only do this conversion if output is still a bitmap container. - if output.typ == containerBitmap && output.count() < ArrayMaxSize { - output.bitmapToArray() + if output.typ() == containerBitmap && output.count() < ArrayMaxSize { + output = output.bitmapToArray() } return output @@ -3560,9 +3801,9 @@ func xorBitmapBitmap(a, b *Container) *Container { n += int32(popcount(ob[i])) } - output := NewContainerBitmap(n, ob) - if output.count() < ArrayMaxSize { - output.bitmapToArray() + output := NewContainerBitmapN(ob, n) + if n < ArrayMaxSize { + output = output.bitmapToArray() } return output } @@ -3571,6 +3812,9 @@ func xorBitmapBitmap(a, b *Container) *Container { // the new container and a bool indicating whether a // carry bit was shifted out. func shift(c *Container) (*Container, bool) { + if c.N() == 0 { + return nil, false + } if c.isArray() { return shiftArray(c) } else if c.isRun() { @@ -3598,23 +3842,18 @@ func shiftArray(a *Container) (*Container, bool) { // shiftBitmap is a bitmap-specific implementation of shift(). func shiftBitmap(a *Container) (*Container, bool) { statsHit("shift/Bitmap") - carry := false - output := NewContainerBitmap(a.n, nil) + carry := uint64(0) + output := NewContainerBitmapN(nil, 0) ba, bo := a.bitmap(), output.bitmap() - lastCarry := false + lastCarry := uint64(0) for i, v := range ba { - carry = (v & (1 << 63)) != 0 - v = v << 1 - if lastCarry { - v |= 1 - } + carry = v >> 63 + v = v<<1 | lastCarry bo[i] = v lastCarry = carry } - if carry { - output.n-- - } - return output, carry + output.setN(a.N() - int32(carry)) + return output, carry != 0 } // shiftRun is a run-specific implementation of shift(). @@ -3695,8 +3934,8 @@ func (op *op) WriteTo(w io.Writer) (n int64, err error) { // Add checksum at the end. h := fnv.New32a() - h.Write(buf[0:9]) - h.Write(buf[13:]) + _, _ = h.Write(buf[0:9]) + _, _ = h.Write(buf[13:]) binary.LittleEndian.PutUint32(buf[9:13], h.Sum32()) // Write to writer. @@ -3719,13 +3958,13 @@ func (op *op) UnmarshalBinary(data []byte) error { // Verify checksum. h := fnv.New32a() - h.Write(data[0:9]) + _, _ = h.Write(data[0:9]) if op.typ > 1 { if len(data) < int(13+op.value*8) { return fmt.Errorf("op data truncated - expected %d, got %d", 13+op.value*8, len(data)) } - h.Write(data[13 : 13+op.value*8]) + _, _ = h.Write(data[13 : 13+op.value*8]) op.values = make([]uint64, op.value) for i := uint64(0); i < op.value; i++ { start := 13 + i*8 @@ -3896,6 +4135,7 @@ func xorArrayRun(a, b *Container) *Container { var vb interval16 var va uint16 lastI, lastJ := -1, -1 + n := int32((0)) for i, j := 0, 0; i < na || j < nb; { if i < na && i != lastI { va = aa[i] @@ -3907,14 +4147,14 @@ func xorArrayRun(a, b *Container) *Container { lastJ = j if i < na && (j >= nb || va < vb.start) { //before - output.n += output.runAppendInterval(interval16{start: va, last: va}) + n += output.runAppendInterval(interval16{start: va, last: va}) i++ } else if j < nb && (i >= na || va > vb.last) { //after - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else if va > vb.start { if va < vb.last { - output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) + n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) i++ vb.start = va + 1 @@ -3922,12 +4162,12 @@ func xorArrayRun(a, b *Container) *Container { j++ } } else if va > vb.last { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) j++ } else { // va == vb.last vb.last-- if vb.start <= vb.last { - output.n += output.runAppendInterval(vb) + n += output.runAppendInterval(vb) } j++ i++ @@ -3945,10 +4185,11 @@ func xorArrayRun(a, b *Container) *Container { i++ } } - if output.n < ArrayMaxSize { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize { + output = output.runToArray() } else if len(output.runs()) > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -4052,18 +4293,13 @@ func xorRunRun(a, b *Container) *Container { statsHit("xor/RunRun") ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) - if na == 0 { - return b.Clone() - } - if nb == 0 { - return a.Clone() - } output := NewContainerRun(nil) lastI, lastJ := -1, -1 state := &xorstm{} + n := int32(0) for i, j := 0, 0; i < na || j < nb; { if i < na && lastI != i { state.va = ra[i] @@ -4078,7 +4314,7 @@ func xorRunRun(a, b *Container) *Container { r1, ok := xorCompare(state) if ok { - output.n += output.runAppendInterval(r1) + n += output.runAppendInterval(r1) } if !state.vaValid { i++ @@ -4090,10 +4326,11 @@ func xorRunRun(a, b *Container) *Container { } l := len(output.runs()) - if output.n < ArrayMaxSize && int32(l) > output.n/2 { - output.runToArray() + output.setN(n) + if n < ArrayMaxSize && int32(l) > n/2 { + output = output.runToArray() } else if l > runMaxSize { - output.runToBitmap() + output = output.runToBitmap() } return output } @@ -4163,11 +4400,11 @@ const ( serialCookie = 12347 // runs, arrays, and bitmaps ) -func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { +func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, flags byte, haveRuns bool, err error) { statsHit("readOfficialHeader") if len(buf) < 8 { err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } cf := func(index uint, card int) (newType byte) { newType = containerBitmap @@ -4177,7 +4414,8 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint return newType } containerTyper = cf - cookie := binary.LittleEndian.Uint32(buf) + cookie := binary.LittleEndian.Uint32(buf) & 0xFFFFFF + flags = buf[3] pos += 4 // cookie header @@ -4192,7 +4430,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint isRunBitmapSize := (int(size) + 7) / 8 if pos+isRunBitmapSize > len(buf) { err = fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } isRunBitmap := buf[pos : pos+isRunBitmapSize] @@ -4205,22 +4443,22 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint } } else { err = fmt.Errorf("did not find expected serialCookie in header") - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } header = pos if size > (1 << 16) { err = fmt.Errorf("it is logically impossible to have more than (1<<16) containers") - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } // descriptive header if pos+2*2*int(size) > len(buf) { err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } pos += 2 * 2 * int(size) // moving pos past keycount - return size, containerTyper, header, pos, haveRuns, err + return size, containerTyper, header, pos, flags, haveRuns, err } // UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in @@ -4237,10 +4475,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } - keyN, containerTyper, header, pos, haveRuns, err := readOfficialHeader(data) + keyN, containerTyper, header, pos, flags, haveRuns, err := readOfficialHeader(data) if err != nil { return errors.Wrap(err, "reading roaring header") } + b.Flags = flags b.Containers.Reset() // Descriptive header section: Read container keys and cardinalities. @@ -4278,13 +4517,13 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error { // Map byte slice directly to the container data. citer.Next() _, c := citer.Value() - switch c.typ { + switch c.typ() { case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n:c.n]) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) default: - return fmt.Errorf("unsupported container type %d", c.typ) + return fmt.Errorf("unsupported container type %d", c.typ()) } } return nil @@ -4295,7 +4534,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { for i := 0; i < int(keyN); i++ { citer.Next() _, c := citer.Value() - switch c.typ { + switch c.typ() { case containerRun: runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount:runCount]) @@ -4306,8 +4545,8 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } pos += int((runCount * interval16Size) + runCountHeaderSize) case containerArray: - c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n:c.n]) - pos += int(c.n * 2) + c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.N():c.N()]) + pos += int(c.N() * 2) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN:bitmapN]) pos += bitmapN * 8 @@ -4362,9 +4601,9 @@ func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummarySta if key == currKey { summary.c++ - summary.n += int64(currContainer.n) + summary.n += int64(currContainer.N()) - if currContainer.n == maxContainerVal+1 { + if currContainer.N() == maxContainerVal+1 { summary.hasMaxRange = true summary.n = maxContainerVal + 1 return summary diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 8c8bbd9ac..41d4059fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -253,8 +253,7 @@ func doContainer(typ byte, data interface{}) *Container { case containerArray: return NewContainerArray(data.([]uint16)) case containerBitmap: - c := NewContainerBitmap(0, data.([]uint64)) - c.n = c.count() + c := NewContainerBitmap(-1, data.([]uint64)) return c case containerRun: return NewContainerRun(data.([]interval16)) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index e88356a37..95c8d1b0b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -32,10 +32,6 @@ func (iv interval16) String() string { return fmt.Sprintf("[%d, %d]", iv.start, iv.last) } -func (c *Container) String() string { - return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%s", c.info().Type, c.n, len(c.array()), len(c.runs()), len(c.bitmap()), containerTypeNames[c.typ]) -} - func TestRunAppendInterval(t *testing.T) { a := NewContainerRun(nil) tests := []struct { @@ -102,15 +98,15 @@ func TestContainerRunAdd(t *testing.T) { {8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}}, } for _, test := range tests { - c.mapped = true - ret := c.add(test.op) - if !ret { + c.setMapped(true) + c, changed := c.add(test.op) + if !changed { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } if !reflect.DeepEqual(c.runs(), test.exp) { t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs(), test.op) } - if c.mapped { + if c.Mapped() { t.Fatalf("container should not be mapped after adding bit %v", test.op) } } @@ -118,14 +114,14 @@ func TestContainerRunAdd(t *testing.T) { func TestContainerRunAdd2(t *testing.T) { c := NewContainerRun(nil) - ret := c.add(0) + c, ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } if !reflect.DeepEqual(c.runs(), []interval16{{start: 0, last: 0}}) { t.Fatalf("should have 1 run of length 1, but have %v", c.runs()) } - ret = c.add(0) + c, ret = c.add(0) if ret { t.Fatalf("result of adding existing bit should be false: %v", c.runs()) } @@ -237,23 +233,23 @@ func TestBitmapCountRange(t *testing.T) { } func TestIntersectionCountArrayBitmap3(t *testing.T) { - a, b := NewContainerBitmap(maxContainerVal+1, getFullBitmap()), NewContainerBitmap(maxContainerVal+1, getFullBitmap()) + a, b := NewContainerBitmapN(getFullBitmap(), maxContainerVal+1), NewContainerBitmapN(getFullBitmap(), maxContainerVal+1) res := intersectBitmapBitmap(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 { - t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 { + t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } - a.bitmapToRun(0) + a = a.bitmapToRun(0) res = intersectBitmapRun(b, a) - if res.n != res.count() || res.n != maxContainerVal+1 { - t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 { + t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } b.bitmapToRun(0) res = intersectRunRun(a, b) n := intersectionCountRunRun(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) { - t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) + if res.N() != res.count() || res.N() != maxContainerVal+1 || res.N() != int32(n) { + t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.N(), res.count(), maxContainerVal+1) } } @@ -321,15 +317,16 @@ func TestRunRemove(t *testing.T) { } for i, test := range tests { - c.mapped = true - ret := c.remove(test.op) + c = c.Freeze() + var ret bool + c, ret = c.remove(test.op) if ret != test.expRet || !reflect.DeepEqual(c.runs(), test.exp) { t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs()) } - if ret && c.mapped { + if ret && c.frozen() { t.Fatalf("test #%v container was not unmapped although bit %v was removed", i, test.op) } - if !ret && !c.mapped { + if !ret && !c.frozen() { t.Fatalf("test #%v container was unmapped although bit %v was not removed", i, test.op) } } @@ -370,7 +367,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap(), b.runs(), ret) } - a = NewContainerBitmap(29, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) + a = NewContainerBitmap(-1, []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}) b = NewContainerRun([]interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}) ret = intersectionCountBitmapRun(a, b) @@ -380,8 +377,6 @@ func TestIntersectionCountBitmapRun(t *testing.T) { } func TestIntersectionCountRunRun(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -419,10 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.typ = containerRun - b.typ = containerRun - a.setRuns(test.aruns) - b.setRuns(test.bruns) + a := NewContainerRun(test.aruns) + b := NewContainerRun(test.bruns) ret := intersectionCountRunRun(a, b) if ret != test.exp { t.Fatalf("test #%v failed intersecting %v with %v should be %v, but got %v", i, test.aruns, test.bruns, test.exp, ret) @@ -524,8 +517,8 @@ func TestIntersectRunRun(t *testing.T) { a.setRuns(test.aruns) b.setRuns(test.bruns) ret := intersectRunRun(a, b) - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } if test.exp != nil { if !reflect.DeepEqual(ret.runs(), test.exp) { @@ -538,8 +531,6 @@ func TestIntersectRunRun(t *testing.T) { } func TestIntersectBitmapRunBitmap(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -578,13 +569,11 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - b.setRuns(test.runs) - b.n = 4097 // ;) exp := make([]uint64, bitmapN) copy(exp, test.exp) - a.typ = containerBitmap - b.typ = containerRun + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) + b.setN(4097) ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -592,8 +581,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { if !reflect.DeepEqual(ret.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap()) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } } @@ -646,8 +635,8 @@ func TestIntersectBitmapRunArray(t *testing.T) { if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } } @@ -738,8 +727,7 @@ func TestDifferenceMixed(t *testing.T) { b := NewContainerArray([]uint16{0, 2, 4, 6, 8, 10, 12}) - c := NewContainerBitmap(0, MakeBitmap([]uint64{0x64})) - c.n = c.countRange(0, 100) + c := NewContainerBitmap(-1, MakeBitmap([]uint64{0x64})) d := NewContainerArray([]uint16{1, 3, 5, 7, 9, 11, 12}) @@ -780,13 +768,13 @@ func TestDifferenceMixed(t *testing.T) { } res = difference(b, b) - if res.n != 0 { - t.Fatalf("test #8 expected 0, but got %d", res.n) + if res.N() != 0 { + t.Fatalf("test #8 expected 0, but got %d", res.N()) } res = difference(c, c) - if res.n != 0 { - t.Fatalf("test #9 expected 0, but got %d", res.n) + if res.N() != 0 { + t.Fatalf("test #9 expected 0, but got %d", res.N()) } res = difference(d, b) @@ -901,7 +889,6 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -926,21 +913,18 @@ func TestBitmapSetRange(t *testing.T) { } for i, test := range tests { - bitmap := c.bitmap() - copy(bitmap, test.bitmap) - c.n = c.countRange(0, 65535) - c.bitmapSetRange(bitmap, test.start, test.last+1) + c := NewContainerBitmap(-1, test.bitmap) + c.bitmapSetRange(test.start, test.last+1) if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } } } func TestArrayToBitmap(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint64 @@ -958,19 +942,15 @@ func TestArrayToBitmap(t *testing.T) { for i, test := range tests { exp := make([]uint64, bitmapN) copy(exp, test.exp) - - a.setArray(test.array) - a.n = int32(len(test.array)) - a.arrayToBitmap() + a := NewContainerArray(test.array) + a = a.arrayToBitmap() if !reflect.DeepEqual(a.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } - a.bitmapToArray() } } func TestBitmapToArray(t *testing.T) { - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint16 @@ -985,25 +965,16 @@ func TestBitmapToArray(t *testing.T) { }, } for i, test := range tests { - a.setBitmap(make([]uint64, bitmapN)) - bitmap := a.bitmap() - n := int32(0) - for i, v := range test.bitmap { - bitmap[i] = v - n += int32(popcount(v)) - } - a.n = n + a := NewContainerBitmap(-1, test.bitmap) - a.bitmapToArray() + a = a.bitmapToArray() if !reflect.DeepEqual(a.array(), test.exp) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, a.array()) } - a.arrayToBitmap() } } func TestRunToBitmap(t *testing.T) { - a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint64 @@ -1037,11 +1008,8 @@ func TestRunToBitmap(t *testing.T) { exp[i] = v n += int(popcount(v)) } - - a.typ = containerRun - a.setRuns(test.runs) - a.n = int32(n) - a.runToBitmap() + a := NewContainerRun(test.runs) + a = a.runToBitmap() if !reflect.DeepEqual(a.bitmap(), exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap()) } @@ -1058,7 +1026,6 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []interval16 @@ -1116,15 +1083,8 @@ func TestBitmapToRun(t *testing.T) { tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF for i, test := range tests { - a.setBitmap(make([]uint64, bitmapN)) - bitmap := a.bitmap() - n := 0 - for i, v := range test.bitmap { - bitmap[i] = v - n += int(popcount(v)) - } - a.n = int32(n) - x := bitmap + a := NewContainerBitmap(-1, test.bitmap) + x := a.bitmap() a.bitmapToRun(0) if !reflect.DeepEqual(a.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) @@ -1137,7 +1097,6 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []interval16 @@ -1161,10 +1120,8 @@ func TestArrayToRun(t *testing.T) { } for i, test := range tests { - a.typ = containerArray - a.setArray(test.array) - a.n = int32(len(test.array)) - a.arrayToRun(0) + a := NewContainerArray(test.array) + a = a.arrayToRun(0) if !reflect.DeepEqual(a.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs()) } @@ -1172,7 +1129,6 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := NewContainerRun(nil) tests := []struct { runs []interval16 exp []uint16 @@ -1196,10 +1152,8 @@ func TestRunToArray(t *testing.T) { } for i, test := range tests { - a.typ = containerRun - a.setRuns(test.runs) - a.n = int32(len(test.exp)) - a.runToArray() + a := NewContainerRun(test.runs) + a = a.runToArray() if !reflect.DeepEqual(a.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array()) } @@ -1207,7 +1161,6 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -1230,17 +1183,16 @@ func TestBitmapZeroRange(t *testing.T) { expN: 13, }, } - bitmap := c.bitmap() for i, test := range tests { - copy(bitmap, test.bitmap) - c.n = c.countRange(0, 65535) + c := NewContainerBitmap(-1, test.bitmap) + bitmap := c.bitmap() c.bitmapZeroRange(test.start, test.last+1) if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, bitmap[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } for i := range test.bitmap { bitmap[i] = 0 @@ -1250,8 +1202,6 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1266,20 +1216,18 @@ func TestUnionBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - a.n = a.bitmapCountRange(0, 65535) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 65535) + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) ret := unionBitmapRun(a, b) if ret.isArray() { - ret.arrayToBitmap() + ret = ret.arrayToBitmap() } bitmap := ret.bitmap() if !reflect.DeepEqual(bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected %x, but got %x", i, test.exp, bitmap[:len(test.exp)]) } - if ret.n != test.expN { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) + if ret.N() != test.expN { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.N()) } for i := range test.bitmap { a.bitmap()[i] = 0 @@ -1288,7 +1236,6 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp int32 @@ -1310,10 +1257,10 @@ func TestBitmapCountRuns(t *testing.T) { exp: 10, }, } + var c *Container for i, test := range tests { - copy(c.bitmap(), test.bitmap) - + c = NewContainerBitmap(-1, test.bitmap) ret := c.bitmapCountRuns() if ret != test.exp { t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) @@ -1377,8 +1324,6 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := NewContainerArray(nil) - b := NewContainerRun(nil) tests := []struct { array []uint16 runs []interval16 @@ -1391,10 +1336,8 @@ func TestDifferenceArrayRun(t *testing.T) { }, } for i, test := range tests { - a.setArray(test.array) - a.n = int32(len(a.array())) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 100) + a := NewContainerArray(test.array) + b := NewContainerRun(test.runs) ret := differenceArrayRun(a, b) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) @@ -1403,8 +1346,6 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerArray(nil) tests := []struct { runs []interval16 array []uint16 @@ -1457,10 +1398,8 @@ func TestDifferenceRunArray(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.runs) - a.n = a.runCountRange(0, 100) - b.setArray(test.array) - b.n = int32(len(b.array())) + a := NewContainerRun(test.runs) + b := NewContainerArray(test.array) ret := differenceRunArray(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) @@ -1480,8 +1419,6 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerBitmap(0, nil) tests := []struct { runs []interval16 bitmap []uint64 @@ -1529,10 +1466,8 @@ func TestDifferenceRunBitmap(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.runs) - a.n = a.runCountRange(0, 65536) - copy(b.bitmap(), test.bitmap) - b.n = b.bitmapCountRange(0, 65536) + a := NewContainerRun(test.runs) + b := NewContainerBitmap(-1, test.bitmap) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) @@ -1541,8 +1476,6 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerRun(nil) tests := []struct { bitmap []uint64 runs []interval16 @@ -1610,10 +1543,8 @@ func TestDifferenceBitmapRun(t *testing.T) { }, } for i, test := range tests { - copy(a.bitmap(), test.bitmap) - a.n = a.bitmapCountRange(0, 65536) - b.setRuns(test.runs) - b.n = b.runCountRange(0, 65536) + a := NewContainerBitmap(-1, test.bitmap) + b := NewContainerRun(test.runs) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap()[:len(test.exp)]) @@ -1622,8 +1553,6 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := NewContainerBitmap(0, nil) - a := NewContainerArray(nil) tests := []struct { bitmap []uint64 array []uint16 @@ -1661,9 +1590,8 @@ func TestDifferenceBitmapArray(t *testing.T) { }, } for i, test := range tests { - b.bitmap()[0] = test.bitmap[0] - b.n = b.count() - a.setArray(test.array) + b := NewContainerBitmap(-1, test.bitmap[:1]) + a := NewContainerArray(test.array) ret := differenceBitmapArray(b, a) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret.array()) @@ -1672,8 +1600,6 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := NewContainerBitmap(0, nil) - b := NewContainerBitmap(0, nil) tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1691,9 +1617,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { }, } for i, test := range tests { - a.bitmap()[0] = test.abitmap[0] - b.bitmap()[0] = test.bbitmap[0] - + a := NewContainerBitmap(-1, test.abitmap) + b := NewContainerBitmap(-1, test.bbitmap) ret := differenceBitmapBitmap(a, b) if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array()) @@ -1702,8 +1627,6 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := NewContainerRun(nil) - b := NewContainerRun(nil) tests := []struct { aruns []interval16 bruns []interval16 @@ -1721,16 +1644,14 @@ func TestDifferenceRunRun(t *testing.T) { }, } for i, test := range tests { - a.setRuns(test.aruns) - a.n = a.runCountRange(0, 100) - b.setRuns(test.bruns) - b.n = b.runCountRange(0, 100) + a := NewContainerRun(test.aruns) + b := NewContainerRun(test.bruns) ret := differenceRunRun(a, b) if !reflect.DeepEqual(ret.runs(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } - if ret.n != test.expn { - t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.n) + if ret.N() != test.expn { + t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.N()) } } } @@ -1756,7 +1677,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := NewContainerBitmap(129*32, nil) + cb := NewContainerBitmapN(nil, 129*32) for i := 0; i < 129; i++ { cb.bitmap()[i] = 0x5555555555555555 } @@ -1779,7 +1700,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := NewContainerBitmap(65536, nil) + cb := NewContainerBitmapN(nil, 65536) for i := 0; i < bitmapN; i++ { cb.bitmap()[i] = 0xffffffffffffffff } @@ -1802,11 +1723,11 @@ func TestWriteReadFullBitmap(t *testing.T) { t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap(), bb2.Containers.Get(0).bitmap()) } - if bb2.Containers.Get(0).n != cb.n { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) + if bb2.Containers.Get(0).N() != cb.N() { + t.Fatalf("bitmap test expected count %x, but got %x", cb.N(), bb2.Containers.Get(0).N()) } if bb2.Containers.Get(0).count() != cb.count() { - t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.Containers.Get(0).n) + t.Fatalf("bitmap test expected count %x, but got %x", cb.N(), bb2.Containers.Get(0).N()) } } @@ -1855,8 +1776,8 @@ func TestXorArrayRun(t *testing.T) { } for i, test := range tests { - test.a.n = test.a.count() - test.b.n = test.b.count() + test.a.setN(test.a.count()) + test.b.setN(test.b.count()) ret := xor(test.a, test.b) if !reflect.DeepEqual(ret.array(), test.exp.array()) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret) @@ -1982,7 +1903,6 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 start uint64 @@ -2035,14 +1955,13 @@ func TestBitmapXorRange(t *testing.T) { } for i, test := range tests { - copy(c.bitmap(), test.bitmap) - c.n = c.countRange(0, 65535) + c := NewContainerBitmap(-1, test.bitmap) c.bitmapXorRange(test.start, test.last+1) if !reflect.DeepEqual(c.bitmap()[:len(test.exp)], test.exp) { t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap()[:len(test.bitmap)]) } - if test.expN != c.n { - t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) + if test.expN != c.N() { + t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.N()) } } } @@ -2060,15 +1979,21 @@ func TestXorBitmapRun(t *testing.T) { }, } for i, test := range tests { - a := NewContainerBitmap(0, test.bitmap) - e := NewContainerBitmap(0, test.exp) + a := NewContainerBitmap(-1, test.bitmap) + e := NewContainerBitmap(-1, test.exp) b := NewContainerRun(test.runs) //xorBitmapRun ret := xor(a, b) + if ret.isRun() { + ret = ret.runToBitmap() + } if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { t.Fatalf("test #%v expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } ret = xor(b, a) + if ret.isRun() { + ret = ret.runToBitmap() + } if !reflect.DeepEqual(ret.bitmap(), e.bitmap()) { t.Fatalf("test #%v.1 expected %v, but got %v", i, e.bitmap(), ret.bitmap()) } @@ -2127,10 +2052,14 @@ func TestIteratorBitmap(t *testing.T) { // but won't update to RLE until Optimize() is called b := NewFileBitmap() for i := uint64(61000); i < 71000; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } for i := uint64(75000); i < 75100; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } if !b.Containers.Get(0).isBitmap() { t.Fatalf("wrong container type") @@ -2393,26 +2322,31 @@ func TestRunBinSearch(t *testing.T) { } func TestBitmap_RemoveEmptyContainers(t *testing.T) { bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) - bm1.Remove(2 << 16) - if bm1.countEmptyContainers() != 1 { + bm2 := NewFileBitmap(1<<16, 2<<16+1, 3<<16) + bm3 := bm1.Intersect(bm2) + if bm3.countEmptyContainers() != 1 { t.Fatalf("Should be 1 empty container ") } - bm1.removeEmptyContainers() + bm3.removeEmptyContainers() - if bm1.countEmptyContainers() != 0 { + if bm3.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") } } func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { bm1 := NewFileBitmap(1<<16, 2<<16, 3<<16) - bm1.Remove(2 << 16) + if _, err := bm1.Remove(2 << 16); err != nil { + t.Fatalf("removing a bit: %v", err) + } var buf bytes.Buffer if _, err := bm1.WriteTo(&buf); err != nil { t.Fatalf("Failure to write to bitmap buffer. ") } bm0 := NewFileBitmap() - bm0.UnmarshalBinary(buf.Bytes()) + if err := bm0.UnmarshalBinary(buf.Bytes()); err != nil { + t.Fatalf("unmarshalling: %v", err) + } if bm0.countEmptyContainers() != 0 { t.Fatalf("Should be no empty containers ") } @@ -2500,7 +2434,6 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := NewContainerArray(nil), NewContainerBitmap(0, nil) tests := []struct { array []uint16 bitmap []uint64 @@ -2544,8 +2477,8 @@ func TestIntersectArrayBitmap(t *testing.T) { } for i, test := range tests { - a.setArray(test.array) - copy(b.bitmap(), test.bitmap) + a := NewContainerArray(test.array) + b := NewContainerBitmap(-1, test.bitmap) ret := intersectArrayBitmap(a, b).array() if len(ret) == 0 && len(test.exp) == 0 { continue @@ -2559,7 +2492,9 @@ func TestIntersectArrayBitmap(t *testing.T) { func TestBitmapClone(t *testing.T) { b := NewFileBitmap() for i := uint64(61000); i < 71000; i++ { - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bit: %v", err) + } } c := b.Clone() if err := bitmapsEqual(b, c); err != nil { @@ -3262,42 +3197,48 @@ func TestContainerCombinations(t *testing.T) { for _, ct := range containerTypes { clone := ret.Clone() if ct == containerArray { - if clone.isBitmap() { - clone.bitmapToArray() + if clone == nil { + clone = NewContainerArray(nil) + } else if clone.isBitmap() { + clone = clone.bitmapToArray() } else if clone.isRun() { - clone.runToArray() + clone = clone.runToArray() } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } // Because xorRunRun resulting in an empty container returns an array container with a // nil slice array, then we need to check len() on array first (look for 0). if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) { - t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) + t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) } } else if ct == containerBitmap { - if clone.isArray() { - clone.arrayToBitmap() + if clone == nil { + clone = NewContainerBitmap(0, nil) + } else if clone.isArray() { + clone = clone.arrayToBitmap() } else if clone.isRun() { - clone.runToBitmap() + clone = clone.runToBitmap() } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) { - t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) + t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) } } else if ct == containerRun { - if clone.isArray() { - clone.arrayToRun(0) + if clone == nil { + clone = NewContainerRun(nil) + } else if clone.isArray() { + clone = clone.arrayToRun(0) } else if clone.isBitmap() { - clone.bitmapToRun(0) + clone = clone.bitmapToRun(0) } - if clone.n != cts[ct][exp].n { - t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + if clone.N() != cts[ct][exp].N() { + t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) } if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) { - t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) + t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) } } } @@ -3393,7 +3334,6 @@ func TestEquals(t *testing.T) { } */ func TestShiftArray(t *testing.T) { - a := NewContainerArray(nil) tests := []struct { array []uint16 exp []uint16 @@ -3417,10 +3357,16 @@ func TestShiftArray(t *testing.T) { } for i, test := range tests { - a.setArray(test.array) - a.n = int32(len(a.array())) + a := NewContainerArray(test.array) ret1, _ := shift(a) // test generic shift function ret2, _ := shiftArray(a) // test array-specific shift function + // accept nil *Container as valid substitute for empty array + if ret1 == nil { + ret1 = NewContainerArray(nil) + } + if ret2 == nil { + ret2 = NewContainerArray(nil) + } if !reflect.DeepEqual(ret1.array(), test.exp) { t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array()) } else if !reflect.DeepEqual(ret2.array(), test.exp) { @@ -3431,7 +3377,6 @@ func TestShiftArray(t *testing.T) { func TestShiftBitmap(t *testing.T) { // note, bitmaps are provided for us by the ensuing tests - a := NewContainerBitmap(0, nil) tests := []struct { bitmap []uint64 exp []uint64 @@ -3451,11 +3396,10 @@ func TestShiftBitmap(t *testing.T) { } for i, test := range tests { - a.setBitmap(test.bitmap) - a.n = 1 + a := NewContainerBitmap(-1, test.bitmap) ret1, _ := shift(a) // test generic shift function ret2, _ := shiftBitmap(a) // test bitmap-specific shift function - e := NewContainerBitmap(1, test.exp) + e := NewContainerBitmap(-1, test.exp) if !reflect.DeepEqual(ret1.bitmap(), e.bitmap()) { t.Fatalf("test #%v shift() expected %v, but got %v", i, e.bitmap(), ret1.bitmap()) } else if !reflect.DeepEqual(ret2.bitmap(), test.exp) { @@ -3464,8 +3408,6 @@ func TestShiftBitmap(t *testing.T) { } } func TestShiftRun(t *testing.T) { - a := NewContainerRun(nil) - tests := []struct { runs []interval16 n int32 @@ -3497,14 +3439,13 @@ func TestShiftRun(t *testing.T) { } for i, test := range tests { - a.setRuns(test.runs) - a.n = test.n + a := NewContainerRun(test.runs) ret1, c1 := shift(a) // test generic shift function ret2, c2 := shiftRun(a) // test run-specific shift function - if !reflect.DeepEqual(ret1.runs(), test.exp) && c1 == test.carry && ret1.n == test.en { - t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs(), ret1.n) - } else if !reflect.DeepEqual(ret2.runs(), test.exp) && c2 == test.carry && ret2.n == test.en { - t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs(), ret2.n) + if !reflect.DeepEqual(ret1.runs(), test.exp) && c1 == test.carry && ret1.N() == test.en { + t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs(), ret1.N()) + } else if !reflect.DeepEqual(ret2.runs(), test.exp) && c2 == test.carry && ret2.N() == test.en { + t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs(), ret2.N()) } } } @@ -3770,24 +3711,45 @@ func TestBitmapAny(t *testing.T) { if bm.Any() { t.Error("empty bitmap should have Any()==false") } - bm.Add(1) + _, err := bm.Add(1) + if err != nil { + t.Errorf("couldn't add a bit: %v", err) + } if !bm.Any() { t.Error("bitmap with 1 bit should have Any()==true") } - bm.Add(100000) + _, err = bm.Add(100000) + if err != nil { + t.Errorf("couldn't add a bit: %v", err) + } if !bm.Any() { t.Error("bitmap with 2 bits should have Any()==true") } - bm.Remove(1) + changed, err := bm.Remove(1) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } + if changed != true { + t.Error("removing a set bit should have been a change") + } if !bm.Any() { t.Error("bitmap with 1 bit left after removing 1 should have Any()==true") } - bm.Add(1) + _, err = bm.Add(1) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } + if changed != true { + t.Error("re-addintg a previously set bit should have been a change") + } bm = bm.Difference(NewBTreeBitmap(1)) if !bm.Any() { t.Error("bitmap with 1 bit left after differencing 1 should have Any()==true") } - bm.Remove(100000) + _, err = bm.Remove(100000) + if err != nil { + t.Errorf("couldn't remove a bit: %v", err) + } if bm.Any() { t.Error("shouldn't be any left") } diff --git a/roaring/roaring_nop_paranoia.go b/roaring/roaring_nop_paranoia.go index ce9af8d54..6f2fa354b 100644 --- a/roaring/roaring_nop_paranoia.go +++ b/roaring/roaring_nop_paranoia.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !roaringparanoia package roaring diff --git a/roaring/roaring_nop_stats.go b/roaring/roaring_nop_stats.go index c9e029ab7..2b1506953 100644 --- a/roaring/roaring_nop_stats.go +++ b/roaring/roaring_nop_stats.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !roaringstats package roaring diff --git a/roaring/roaring_paranoia.go b/roaring/roaring_paranoia.go index 910e1bd40..6f7d0c57e 100644 --- a/roaring/roaring_paranoia.go +++ b/roaring/roaring_paranoia.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build roaringparanoia package roaring diff --git a/roaring/roaring_stats.go b/roaring/roaring_stats.go index fd8ade91f..046c0f087 100644 --- a/roaring/roaring_stats.go +++ b/roaring/roaring_stats.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build roaringstats package roaring diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 9d0b3c8f8..c6ea63eb0 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -164,11 +164,15 @@ func TestCheckBitmap(t *testing.T) { x := 0 for i := uint64(61000); i < 71000; i++ { x++ - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } for i := uint64(75000); i < 75100; i++ { x++ - b.Add(i) + if _, err := b.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } err := b.Check() if err != nil { @@ -198,7 +202,7 @@ func TestCheckFullRun(t *testing.T) { if i%16384 == 0 { b.Optimize() // convert to runs } - b.Add(i) + _, _ = b.Add(i) } err := b.Check() if err != nil { @@ -238,7 +242,13 @@ func TestBitmap_Contains_Empty(t *testing.T) { // Ensure an empty bitmap does nothing when removing an element. func TestBitmap_Remove_Empty(t *testing.T) { - roaring.NewFileBitmap().Remove(1000) + changed, err := roaring.NewFileBitmap().Remove(1000) + if err != nil { + t.Fatalf("got an error removing a bit from an empty bitmap: %v", err) + } + if changed != false { + t.Fatalf("change reported removing a bit from an empty bitmap") + } } // Ensure a bitmap can return a slice of values. @@ -289,7 +299,9 @@ func TestBitmap_ForEachRange(t *testing.T) { func TestBitmap_Max(t *testing.T) { bm := roaring.NewFileBitmap() for i := uint64(1000); i <= 100000; i++ { - bm.Add(i) + if _, err := bm.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } if v := bm.Max(); v != i { t.Fatalf("max: got=%d; want=%d", v, i) @@ -299,10 +311,10 @@ func TestBitmap_Max(t *testing.T) { // Ensure CountRange is correct even if rangekey is prior to initial container. func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { - s := uint64(2009 * 1048576) - e := uint64(2010 * 1048576) + s := uint64(2009 * pilosa.ShardWidth) + e := uint64(2010 * pilosa.ShardWidth) - start := s + (39314024 % 1048576) + start := s + (39314024 % pilosa.ShardWidth) bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i++ { if (i+1)%4096 == 0 { @@ -310,7 +322,9 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { } else { start += 2 } - bm0.Add(start) + if _, err := bm0.Add(start); err != nil { + t.Fatalf("adding bit: %v", err) + } } a := bm0.Count() r := bm0.CountRange(s, e) @@ -323,9 +337,13 @@ func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { func TestBitmap_BitmapCountRange(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) for i := uint64(628); i < 2683301; i++ { - bm0.Add(i) + if _, err := bm0.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm0.Add(2683307); err != nil { + t.Fatalf("adding bits: %v", err) } - bm0.Add(2683307) if n := bm0.CountRange(1, 2683311); n != 2682674 { t.Fatalf("unexpected n: %d", n) } @@ -389,7 +407,9 @@ func TestBitmap_Intersection(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } result := bm0.Intersect(bm1) @@ -403,9 +423,13 @@ func TestBitmap_Union1(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm1.Add(4000000); err != nil { + t.Fatalf("adding bits: %v", err) } - bm1.Add(4000000) result := bm0.Union(bm1) if n := result.Count(); n != 2682675 { @@ -429,9 +453,13 @@ func TestBitmap_UnionInPlace1(t *testing.T) { result = roaring.NewBitmap() ) for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } + } + if _, err := bm1.Add(4000000); err != nil { + t.Fatalf("adding bits: %v", err) } - bm1.Add(4000000) result.UnionInPlace(bm0, bm1) if n := result.Count(); n != 2682675 { @@ -504,7 +532,9 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { // size of a container to ensure we generate a maxRange container. for x := start; x < (start + 2*(0xffff+1)); x++ { set[uint64(x)] = struct{}{} - bitmap.Add(uint64(x)) + if _, err := bitmap.Add(uint64(x)); err != nil { + t.Fatalf("adding bits: %v", err) + } } } @@ -513,7 +543,9 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { for x := 0; x < numIntsPerBatch; x++ { num := uint64(rng.Intn(maxUint64Val)) set[num] = struct{}{} - bitmap.Add(num) + if _, err := bitmap.Add(num); err != nil { + t.Fatalf("adding bits: %v", err) + } } sets = append(sets, set) @@ -588,12 +620,16 @@ func TestBitmap_IntersectArrayArray(t *testing.T) { func TestBitmap_IntersectBitmapBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 2 { - bm0.Add(i) + if _, err := bm0.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 65536; i += 3 { - bm1.Add(i) + if _, err := bm1.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } result := bm0.Intersect(bm1) @@ -620,7 +656,9 @@ func TestBitmap_IntersectRunRun(t *testing.T) { offset := (runLen / 2) + spaceLen for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) { for j := uint64(0); j < runLen; j++ { - bm2.Add(offset + i + j) + if _, err := bm2.Add(offset + i + j); err != nil { + t.Fatalf("adding bits: %v", err) + } } } bm2.Optimize() // convert to runs @@ -629,7 +667,9 @@ func TestBitmap_IntersectRunRun(t *testing.T) { spaceLen = uint64(1) for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) { for j := uint64(0); j < runLen; j++ { - bm3.Add(i + j) + if _, err := bm3.Add(i + j); err != nil { + t.Fatalf("adding bits: %v", err) + } } } bm3.Optimize() // convert to runs @@ -643,7 +683,7 @@ func TestBitmap_Difference(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() for i := uint64(628); i < 2683301; i++ { - bm1.Add(i) + _, _ = bm1.Add(i) } result := bm0.Difference(bm1) if n := result.Count(); n != 1 { @@ -771,7 +811,7 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } result := bm0.Xor(bm1) @@ -802,11 +842,11 @@ func TestBitmap_Xor_BitmapBitmap(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } for i := uint64(1); i < 10000; i += 2 { - bm0.Add(i) + _, _ = bm0.Add(i) } result := bm0.Xor(bm1) @@ -847,7 +887,9 @@ func TestBitmap_Flip_Bitmap(t *testing.T) { bm := roaring.NewFileBitmap() size := uint64(10000) for i := uint64(0); i < size; i += 2 { - bm.Add(i) + if _, err := bm.Add(i); err != nil { + t.Fatalf("adding bits: %v", err) + } } results := bm.Flip(0, size-1) if n := results.Count(); n != size/2 { @@ -921,7 +963,7 @@ func TestBitmap_IntersectionCount_RunRun(t *testing.T) { func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) { bm0 := roaring.NewFileBitmap() for i := uint64(3); i <= 1000006; i += 2 { - bm0.Add(i) + _, _ = bm0.Add(i) } bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006) bm1.Optimize() // convert to runs @@ -938,7 +980,7 @@ func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap(1, 70, 200, 4097, 4098) bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { - bm1.Add(i) + _, _ = bm1.Add(i) } if n := bm0.IntersectionCount(bm1); n != 3 { @@ -953,15 +995,15 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) { bm0 := roaring.NewFileBitmap() bm1 := roaring.NewFileBitmap() for i := uint64(0); i <= 10000; i += 2 { - bm0.Add(i) - bm1.Add(i + 1) + _, _ = bm0.Add(i) + _, _ = bm1.Add(i + 1) } - bm0.Add(1000) - bm1.Add(1000) + _, _ = bm0.Add(1000) + _, _ = bm1.Add(1000) - bm0.Add(2000) - bm1.Add(2000) + _, _ = bm0.Add(2000) + _, _ = bm1.Add(2000) if n := bm0.IntersectionCount(bm1); n != 2 { t.Fatalf("unexpected n: %d", n) @@ -1005,7 +1047,7 @@ func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, ma // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { - quick.Check(func(a []uint64) bool { + err := quick.Check(func(a []uint64) bool { bm := roaring.NewFileBitmap() m := make(map[uint64]struct{}) @@ -1045,14 +1087,15 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { } // Remove all values in random order. - for _, i := range rand.Perm(len(a)) { - removed, _ := bm.Remove(a[i]) + for i, item := range rand.Perm(len(a)) { + removed, _ := bm.Remove(a[item]) if removed { manual_count-- } //check count if manual_count != bm.Count() { - t.Fatalf("expected bitmap Remove count to be: %d got: %d", manual_count, bm.Count()) + t.Fatalf("removing %d/%d [%d] from bitmap: expected bitmap Remove count to be %d, got %d", + i, len(a), a[item], manual_count, bm.Count()) } } @@ -1067,6 +1110,9 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, false, rand)) }, }) + if err != nil { + t.Fatalf("quick check failed: %v", err) + } } func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) } @@ -1091,7 +1137,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { t.Skip("short") } - quick.Check(func(a0, a1 []uint64) bool { + err := quick.Check(func(a0, a1 []uint64) bool { // Create bitmap with initial values set. bm := roaring.NewFileBitmap(a0...) @@ -1145,6 +1191,9 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { values[1] = reflect.ValueOf(GenerateUint64Slice(100, min, max, sorted, rand)) }, }) + if err != nil { + t.Fatalf("quick check failed: %v", err) + } } // Ensure iterator can iterate over all the values on the bitmap. @@ -1167,13 +1216,13 @@ func TestIterator(t *testing.T) { t.Run("run", func(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 11; i += 1 { - bm1.Add(i) + _, _ = bm1.Add(i) } bm1.Optimize() bm2 := roaring.NewFileBitmap() for i := uint64(0); i < 12; i += 1 { - bm2.Add(i) + _, _ = bm2.Add(i) } bm2.Optimize() @@ -1203,23 +1252,24 @@ func TestIterator(t *testing.T) { // testBM creates a bitmap with 3 containers: array, bitmap, and run. func testBM() *roaring.Bitmap { - + // We should possibly be testing the adds for errors, but we + // don't have a clean way to return an error, so we don't right now. bm := roaring.NewFileBitmap() //the array for i := uint64(0); i < 1024; i += 4 { - bm.Add((1 << 16) + i) + _, _ = bm.Add((1 << 16) + i) } //the bitmap for i := uint64(0); i < 16384; i += 2 { - bm.Add((2 << 16) + i) + _, _ = bm.Add((2 << 16) + i) } //small run for i := uint64(0); i < 1024; i += 1 { - bm.Add((3 << 16) + i) + _, _ = bm.Add((3 << 16) + i) } //large run for i := uint64(0); i < 65535; i += 1 { - bm.Add((4 << 16) + i) + _, _ = bm.Add((4 << 16) + i) } bm.Optimize() //count 75007 @@ -1275,9 +1325,13 @@ func isAllType(b *roaring.Bitmap, typ string) bool { return true } +// getBenchData yields some sample data func getBenchData(tb testing.TB) *benchmarkSampleData { data := &sampleData if data.a1 == nil { + // throughout this, we ignore any errors from bitmap adds, + // because errors in those should result in the Optimize + // pass producing the wrong values, so we can just check there. const max = (1 << 24) / 64 // Build bitmap with array container. @@ -1285,28 +1339,28 @@ func getBenchData(tb testing.TB) *benchmarkSampleData { data.a2 = roaring.NewFileBitmap() // two lists of different lengths for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { - data.a1.Add(uint64(rand.Intn(max))) - data.a2.Add(uint64(rand.Intn(max))) + _, _ = data.a1.Add(uint64(rand.Intn(max))) + _, _ = data.a2.Add(uint64(rand.Intn(max))) } for i, n := 0, roaring.ArrayMaxSize/3; i < n; i++ { - data.a1.Add(uint64(rand.Intn(max))) + _, _ = data.a1.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. data.b = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal/3; i < n; i++ { - data.b.Add(uint64(i * 3)) + _, _ = data.b.Add(uint64(i * 3)) } // build bitmap with run container data.r1 = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { - data.r1.Add(uint64(i)) + _, _ = data.r1.Add(uint64(i)) } // build bitmap with multiple runs data.r2 = roaring.NewFileBitmap() for i, n := 0, MaxContainerVal; i < n; i++ { - data.r2.Add(uint64(i)) + _, _ = data.r2.Add(uint64(i)) // break the runs up, this should produce 16 runs, which // is small enough to make RLE tempting if i&0xfff == 0xfff { @@ -1467,7 +1521,7 @@ func BenchmarkContainerLinear(b *testing.B) { bm := bmMaker() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1483,7 +1537,7 @@ func BenchmarkContainerReverse(b *testing.B) { bm := bmMaker() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1498,7 +1552,7 @@ func BenchmarkContainerColumn(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1514,8 +1568,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { - bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) - bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1532,8 +1586,8 @@ func BenchmarkContainerInsideOut(b *testing.B) { bm := bmMaker() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { - bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) - bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) + _, _ = bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1545,7 +1599,7 @@ func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() for col := uint64(0); col < pilosa.ShardWidth; col++ { - bm.Add(col) + _, _ = bm.Add(col) } } } @@ -1554,9 +1608,9 @@ func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { - bm.Add(col) + _, _ = bm.Add(col) } - bm.Add(0) + _, _ = bm.Add(0) } } @@ -1565,14 +1619,14 @@ func BenchmarkSliceAscendingStriped(b *testing.B) { bm := roaring.NewFileBitmap() l := uint64(pilosa.ShardWidth / 8) for col := uint64(0); col < l; col++ { - bm.Add(l*0 + col) - bm.Add(l*1 + col) - bm.Add(l*2 + col) - bm.Add(l*3 + col) - bm.Add(l*4 + col) - bm.Add(l*5 + col) - bm.Add(l*6 + col) - bm.Add(l*7 + col) + _, _ = bm.Add(l*0 + col) + _, _ = bm.Add(l*1 + col) + _, _ = bm.Add(l*2 + col) + _, _ = bm.Add(l*3 + col) + _, _ = bm.Add(l*4 + col) + _, _ = bm.Add(l*5 + col) + _, _ = bm.Add(l*6 + col) + _, _ = bm.Add(l*7 + col) } } } @@ -1582,14 +1636,14 @@ func BenchmarkSliceDescendingStriped(b *testing.B) { bm := roaring.NewFileBitmap() l := uint64(pilosa.ShardWidth / 8) for col := uint64(l); col < l+1; col-- { - bm.Add(l*7 + col) - bm.Add(l*6 + col) - bm.Add(l*5 + col) - bm.Add(l*4 + col) - bm.Add(l*3 + col) - bm.Add(l*2 + col) - bm.Add(l*1 + col) - bm.Add(l*0 + col) + _, _ = bm.Add(l*7 + col) + _, _ = bm.Add(l*6 + col) + _, _ = bm.Add(l*5 + col) + _, _ = bm.Add(l*4 + col) + _, _ = bm.Add(l*3 + col) + _, _ = bm.Add(l*2 + col) + _, _ = bm.Add(l*1 + col) + _, _ = bm.Add(l*0 + col) } } } diff --git a/row.go b/row.go index 0a8fc0acd..79c918311 100644 --- a/row.go +++ b/row.go @@ -43,6 +43,7 @@ func NewRow(columns ...uint64) *Row { return r } +// IsEmpty returns true if the row doesn't contain any set bits. func (r *Row) IsEmpty() bool { if len(r.segments) == 0 { return true @@ -56,6 +57,12 @@ func (r *Row) IsEmpty() bool { return true } +func (r *Row) Freeze() { + for _, s := range r.segments { + s.Freeze() + } +} + // Merge merges data from other into r. func (r *Row) Merge(other *Row) { var segments []rowSegment @@ -112,6 +119,16 @@ func (r *Row) Intersect(other *Row) *Row { return &Row{segments: segments} } +// Any returns true if row contains any bits. +func (r *Row) Any() bool { + for _, s := range r.segments { + if s.data.Any() { + return true + } + } + return false +} + // Xor returns the xor of r and other. func (r *Row) Xor(other *Row) *Row { var segments []rowSegment @@ -199,15 +216,6 @@ func (r *Row) SetBit(i uint64) (changed bool) { return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } -// clearBit clears the i-th column of the row. -func (r *Row) clearBit(i uint64) (changed bool) { // nolint: unparam - s := r.segment(i / ShardWidth) - if s == nil { - return false - } - return s.ClearBit(i) -} - // Segments returns a list of all segments in the row. func (r *Row) Segments() []rowSegment { return r.segments @@ -235,12 +243,12 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *rowSegment { } // Insert new segment. - r.segments = append(r.segments, rowSegment{data: *roaring.NewBitmap()}) + r.segments = append(r.segments, rowSegment{data: roaring.NewSliceBitmap()}) if i < len(r.segments) { copy(r.segments[i+1:], r.segments[i:]) } r.segments[i] = rowSegment{ - data: *roaring.NewBitmap(), + data: roaring.NewSliceBitmap(), shard: shard, writable: true, } @@ -301,13 +309,17 @@ type rowSegment struct { // Underlying raw bitmap implementation. // This is an mmapped bitmap if writable is false. Otherwise // it is a heap allocated bitmap which can be manipulated. - data roaring.Bitmap + data *roaring.Bitmap writable bool // Bit count n uint64 } +func (s *rowSegment) Freeze() { + s.data.Freeze() +} + // Merge adds chunks from other to s. // Chunks in s are overwritten if they exist in other. func (s *rowSegment) Merge(other *rowSegment) { @@ -321,50 +333,58 @@ func (s *rowSegment) Merge(other *rowSegment) { // IntersectionCount returns the number of intersections between s and other. func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { - return s.data.IntersectionCount(&other.data) + return s.data.IntersectionCount(other.data) } // Intersect returns the itersection of s and other. func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { - data := s.data.Intersect(&other.data) + data := s.data.Intersect(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Union returns the bitwise union of s and other. func (s *rowSegment) Union(other *rowSegment) *rowSegment { - data := s.data.Union(&other.data) + data := s.data.Union(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Difference returns the diff of s and other. func (s *rowSegment) Difference(other *rowSegment) *rowSegment { - data := s.data.Difference(&other.data) + data := s.data.Difference(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } // Xor returns the xor of s and other. func (s *rowSegment) Xor(other *rowSegment) *rowSegment { - data := s.data.Xor(&other.data) + data := s.data.Xor(other.data) + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, } } @@ -375,11 +395,13 @@ func (s *rowSegment) Shift() (*rowSegment, error) { if err != nil { return nil, errors.Wrap(err, "shifting roaring data") } + data.Freeze() return &rowSegment{ - data: *data, - shard: s.shard, - n: data.Count(), + data: data, + shard: s.shard, + n: data.Count(), + writable: true, }, nil } @@ -428,7 +450,11 @@ func (s *rowSegment) ensureWritable() { return } - s.data = *s.data.Clone() + // This doesn't actually clone all the containers, but does clone + // the bitmap itself -- we get a new bitmap, but it just marks the + // containers as frozen and shares them. It's now safe to write to + // this bitmap, but the actual containers are copy-on-write. + s.data = s.data.Freeze() s.writable = true } diff --git a/server.go b/server.go index 3e989e530..ad81aa1b3 100644 --- a/server.go +++ b/server.go @@ -75,6 +75,7 @@ type Server struct { // nolint: maligned dataDir string } +// Holder returns the holder for server. // TODO: have this return an interface for Holder instead of concrete object? func (s *Server) Holder() *Holder { return s.holder @@ -83,6 +84,8 @@ func (s *Server) Holder() *Holder { // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error +// OptServerLogger is a functional option on Server +// used to set the logger. func OptServerLogger(l logger.Logger) ServerOption { return func(s *Server) error { s.logger = l @@ -90,6 +93,8 @@ func OptServerLogger(l logger.Logger) ServerOption { } } +// OptServerReplicaN is a functional option on Server +// used to set the number of replicas. func OptServerReplicaN(n int) ServerOption { return func(s *Server) error { s.cluster.ReplicaN = n @@ -97,6 +102,8 @@ func OptServerReplicaN(n int) ServerOption { } } +// OptServerDataDir is a functional option on Server +// used to set the data directory. func OptServerDataDir(dir string) ServerOption { return func(s *Server) error { s.dataDir = dir @@ -104,6 +111,9 @@ func OptServerDataDir(dir string) ServerOption { } } +// OptServerAttrStoreFunc is a functional option on Server +// used to provide the function to use to generate a new +// attribute store. func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { return func(s *Server) error { s.holder.NewAttrStore = af @@ -111,6 +121,8 @@ func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { } } +// OptServerAntiEntropyInterval is a functional option on Server +// used to set the anti-entropy interval. func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { return func(s *Server) error { s.antiEntropyInterval = interval @@ -118,6 +130,8 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { } } +// OptServerLongQueryTime is a functional option on Server +// used to set long query duration. func OptServerLongQueryTime(dur time.Duration) ServerOption { return func(s *Server) error { s.cluster.longQueryTime = dur @@ -125,6 +139,8 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } +// OptServerMaxWritesPerRequest is a functional option on Server +// used to set the maximum number of writes allowed per request. func OptServerMaxWritesPerRequest(n int) ServerOption { return func(s *Server) error { s.maxWritesPerRequest = n @@ -132,6 +148,8 @@ func OptServerMaxWritesPerRequest(n int) ServerOption { } } +// OptServerMetricInterval is a functional option on Server +// used to set the interval between metric samples. func OptServerMetricInterval(dur time.Duration) ServerOption { return func(s *Server) error { s.metricInterval = dur @@ -139,6 +157,8 @@ func OptServerMetricInterval(dur time.Duration) ServerOption { } } +// OptServerSystemInfo is a functional option on Server +// used to set the system information source. func OptServerSystemInfo(si SystemInfo) ServerOption { return func(s *Server) error { s.systemInfo = si @@ -146,6 +166,8 @@ func OptServerSystemInfo(si SystemInfo) ServerOption { } } +// OptServerGCNotifier is a functional option on Server +// used to set the garbage collection notification source. func OptServerGCNotifier(gcn GCNotifier) ServerOption { return func(s *Server) error { s.gcNotifier = gcn @@ -153,6 +175,8 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } +// OptServerInternalClient is a functional option on Server +// used to set the implementation of InternalClient. func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { s.executor = newExecutor(optExecutorInternalQueryClient(c)) @@ -162,7 +186,7 @@ func OptServerInternalClient(c InternalClient) ServerOption { } } -// DEPRECATED +// OptServerPrimaryTranslateStore has been deprecated. func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { return func(s *Server) error { s.logger.Printf("DEPRECATED: OptServerPrimaryTranslateStore") @@ -170,6 +194,9 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { } } +// OptServerPrimaryTranslateStoreFunc is a functional option on Server +// used to specify the function used to create a new primary translate +// store. func OptServerPrimaryTranslateStoreFunc(tf func(interface{}) TranslateStore) ServerOption { return func(s *Server) error { @@ -178,6 +205,8 @@ func OptServerPrimaryTranslateStoreFunc(tf func(interface{}) TranslateStore) Ser } } +// OptServerStatsClient is a functional option on Server +// used to specify the stats client. func OptServerStatsClient(sc stats.StatsClient) ServerOption { return func(s *Server) error { s.holder.Stats = sc @@ -185,6 +214,8 @@ func OptServerStatsClient(sc stats.StatsClient) ServerOption { } } +// OptServerDiagnosticsInterval is a functional option on Server +// used to specify the duration between diagnostic checks. func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { return func(s *Server) error { s.diagnosticInterval = dur @@ -192,6 +223,8 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } +// OptServerURI is a functional option on Server +// used to set the server URI. func OptServerURI(uri *URI) ServerOption { return func(s *Server) error { s.uri = *uri @@ -199,7 +232,7 @@ func OptServerURI(uri *URI) ServerOption { } } -// OptClusterDisabled tells the server whether to use a static cluster with the +// OptServerClusterDisabled tells the server whether to use a static cluster with the // defined hosts. Mostly used for testing. func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { return func(s *Server) error { @@ -209,6 +242,8 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +// OptServerSerializer is a functional option on Server +// used to set the serializer. func OptServerSerializer(ser Serializer) ServerOption { return func(s *Server) error { s.serializer = ser @@ -216,6 +251,8 @@ func OptServerSerializer(ser Serializer) ServerOption { } } +// OptServerIsCoordinator is a functional option on Server +// used to specify whether or not this server is the coordinator. func OptServerIsCoordinator(is bool) ServerOption { return func(s *Server) error { s.isCoordinator = is @@ -223,6 +260,8 @@ func OptServerIsCoordinator(is bool) ServerOption { } } +// OptServerNodeID is a functional option on Server +// used to set the server node ID. func OptServerNodeID(nodeID string) ServerOption { return func(s *Server) error { s.nodeID = nodeID @@ -230,6 +269,9 @@ func OptServerNodeID(nodeID string) ServerOption { } } +// OptServerClusterHasher is a functional option on Server +// used to specify the consistent hash algorithm for data +// location within the cluster. func OptServerClusterHasher(h Hasher) ServerOption { return func(s *Server) error { s.cluster.Hasher = h @@ -237,6 +279,8 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } +// OptServerTranslateFileMapSize is a functional option on Server +// used to specify the size of the translate file. func OptServerTranslateFileMapSize(mapSize int) ServerOption { return func(s *Server) error { s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) @@ -321,9 +365,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest s.holder.broadcaster = s - s.holder.shardValidatorFunc = func(index string, shard uint64) bool { - return s.cluster.ownsShard(s.nodeID, index, shard) - } err = s.cluster.setup() if err != nil { @@ -559,7 +600,7 @@ func (s *Server) receiveMessage(m Message) error { return err } case *SetCoordinatorMessage: - s.cluster.setCoordinator(obj.New) + return s.cluster.setCoordinator(obj.New) case *UpdateCoordinatorMessage: s.cluster.updateCoordinator(obj.New) case *NodeStateMessage: @@ -681,9 +722,8 @@ func (s *Server) monitorDiagnostics() { if s.diagnosticInterval < time.Minute { s.logger.Printf("diagnostics disabled") return - } else { - s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) } + s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) @@ -705,7 +745,10 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.EnrichWithMemoryInfo() s.diagnostics.EnrichWithSchemaProperties() - s.diagnostics.CheckVersion() + err = s.diagnostics.CheckVersion() + if err != nil { + s.logger.Printf("can't check version: %v", err) + } err = s.diagnostics.Flush() if err != nil { s.logger.Printf("diagnostics error: %s", err) diff --git a/server/cluster_test.go b/server/cluster_test.go index a0cd83f1b..b6f6cf6bf 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -33,8 +33,7 @@ import ( func TestMain_SendReceiveMessage(t *testing.T) { ms := test.MustRunCluster(t, 2) m0, m1 := ms[0], ms[1] - defer m0.Close() - defer m1.Close() + defer ms.Close() // Expected indexes and Fields expected := map[string][]string{ @@ -85,10 +84,10 @@ func TestMain_SendReceiveMessage(t *testing.T) { } // Write data on first node. - if _, err := m0.Query("i", "", ` + if _, err := m0.Query("i", "", fmt.Sprintf(` Set(1, f=1) - Set(2400000, f=1) - `); err != nil { + Set(%d, f=1) + `, 2*pilosa.ShardWidth+1)); err != nil { t.Fatal(err) } @@ -127,8 +126,7 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { clus := test.MustRunCluster(t, 2) - defer clus[0].Close() - defer clus[1].Close() + defer clus.Close() if clus[0].API.State() != pilosa.ClusterStateNormal { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) @@ -141,6 +139,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { clus := test.MustRunCluster(t, 2) + defer clus.Close() if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) @@ -552,6 +551,7 @@ func TestCluster_GossipMembership(t *testing.T) { func TestClusterResize_RemoveNode(t *testing.T) { cluster := test.MustRunCluster(t, 3) + defer cluster.Close() m0 := cluster[0] m1 := cluster[1] diff --git a/server/config_internal_test.go b/server/config_internal_test.go index 2b29ea51e..ca7548895 100644 --- a/server/config_internal_test.go +++ b/server/config_internal_test.go @@ -16,6 +16,7 @@ package server import ( "context" + "fmt" "net" "os" "strings" @@ -125,29 +126,31 @@ func TestConfig_validateAddrs(t *testing.T) { } for i, test := range tests { - c := NewConfig() + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + c := NewConfig() - c.Bind = test.in.bind - c.Advertise = test.in.advertise + c.Bind = test.in.bind + c.Advertise = test.in.advertise - err := c.validateAddrs(context.Background()) + err := c.validateAddrs(context.Background()) - if err != nil && test.expErr == "" { - t.Fatal(err) - } else if err == nil && test.expErr != "" { - t.Fatalf("test %d: expected error string to contain %s, but got no error", i, test.expErr) - } else if err != nil && test.expErr != "" { - if strings.Contains(err.Error(), test.expErr) { - continue - } else { - t.Fatalf("test %d: expected error string to contain %s, but got %s", i, test.expErr, err.Error()) + if err != nil && test.expErr == "" { + t.Fatal(err) + } else if err == nil && test.expErr != "" { + t.Fatalf("expected error string to contain %s, but got no error", test.expErr) + } else if err != nil && test.expErr != "" { + if strings.Contains(err.Error(), test.expErr) { + return + } else { + t.Fatalf("expected error string to contain %s, but got %s", test.expErr, err.Error()) + } } - } - if c.Bind != test.exp.bind { - t.Fatalf("test %d: bind address: expected %s, but got %s", i, test.exp.bind, c.Bind) - } else if c.Advertise != test.exp.advertise { - t.Fatalf("test %d: advertise address: expected %s, but got %s", i, test.exp.advertise, c.Advertise) - } + if c.Bind != test.exp.bind { + t.Fatalf("bind address: expected %s, but got %s", test.exp.bind, c.Bind) + } else if c.Advertise != test.exp.advertise { + t.Fatalf("advertise address: expected %s, but got %s", test.exp.advertise, c.Advertise) + } + }) } } diff --git a/server/enterprise.go b/server/enterprise.go index 6dd3e5540..021c00e7c 100644 --- a/server/enterprise.go +++ b/server/enterprise.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/server/handler_test.go b/server/handler_test.go index f45aa2422..c4e98d7f3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -19,8 +19,10 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "io" "io/ioutil" + "math" gohttp "net/http" "net/http/httptest" "reflect" @@ -35,13 +37,54 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure the handler returns "not found" for invalid paths. +func TestHandler_PostSchemaCluster(t *testing.T) { + cluster := test.MustRunCluster(t, 3) + defer cluster.Close() + cmd := cluster[0] + h := cmd.Handler.(*http.Handler).Handler + + t.Run("PostSchema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) + if w.Code != gohttp.StatusNoContent { + bod, err := ioutil.ReadAll(w.Result().Body) + if err != nil { + t.Errorf("reading body: %v", err) + } + t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod) + } + for i := 0; i < len(cluster); i++ { + cmd = cluster[i] + idx, err := cmd.API.Index(context.Background(), "blah") + if err != nil { + t.Fatalf("getting index: %v", err) + } + if idx.Name() != "blah" { + t.Fatalf("index did not get set, got %v", idx.Name()) + } + + fld, err := cmd.API.Field(context.Background(), "blah", "f1") + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Name() != "f1" { + t.Fatalf("unexpected field: %v", fld.Name()) + } + } + + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/blah", nil)) + }) +} + func TestHandler_Endpoints(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} + // Ensure the handler returns "not found" for invalid paths. t.Run("Not Found", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -50,6 +93,48 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("SchemaEmpty", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := w.Body.String() + if body != "{\"indexes\":null}\n" { + t.Fatalf("unexpected empty schema: '%v'", body) + } + + }) + + t.Run("PostSchema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) + if w.Code != gohttp.StatusNoContent { + bod, err := ioutil.ReadAll(w.Result().Body) + if err != nil { + t.Errorf("reading body: %v", err) + } + t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod) + } + idx, err := cmd.API.Index(context.Background(), "blah") + if err != nil { + t.Fatalf("getting index: %v", err) + } + if idx.Name() != "blah" { + t.Fatalf("index did not get set, got %v", idx.Name()) + } + + fld, err := cmd.API.Field(context.Background(), "blah", "f1") + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Name() != "f1" { + t.Fatalf("unexpected field: %v", fld.Name()) + } + + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/blah", nil)) + }) + t.Run("Info", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) @@ -109,8 +194,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } body := w.Body.String() - target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]} -` + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} +`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } @@ -298,7 +383,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { + } else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" { t.Fatalf("unexpected body: %s", body) } }) @@ -315,10 +400,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ColumnAttrs_JSON", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)"))) + exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n" if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) + } else if body := w.Body.String(); body != exp { + t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp) } }) @@ -457,6 +543,104 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("Query int field unbounded", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int"}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if math.MinInt64 != field.Options.Min { + t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min) + } + if math.MaxInt64 != field.Options.Max { + t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max) + } + }) + + t.Run("Query int field unbounded min", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-min" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "max": 10}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if math.MinInt64 != field.Options.Min { + t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min) + } + if 10 != field.Options.Max { + t.Fatalf("field max %d != %d", 10, field.Options.Max) + } + }) + + t.Run("Query int field unbounded max", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-max" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "min": -10}}`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + rsp := getSchemaResponse{} + if err := json.Unmarshal(w.Body.Bytes(), &rsp); err != nil { + t.Fatalf("json decode: %s", err) + } + field := rsp.findField("i0", fieldName) + if field == nil { + t.Fatalf("field not found: %s", fieldName) + } + if -10 != field.Options.Min { + t.Fatalf("field min %d != %d", 10, field.Options.Min) + } + if math.MaxInt64 != field.Options.Max { + t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max) + } + }) + + t.Run("Query int field min > max return 400", func(t *testing.T) { + w := httptest.NewRecorder() + fieldName := "f-int-ubound-err" + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", fmt.Sprintf("/index/i0/field/%s", fieldName), + strings.NewReader(`{"options":{"type":"int", "min": 10, "max": -10}}`))) + fmt.Println("body", w.Body.String()) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) + t.Run("Method not allowed", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil)) @@ -658,6 +842,7 @@ func TestHandler_Endpoints(t *testing.T) { } clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) + defer clus.Close() w = httptest.NewRecorder() h := clus[0].Handler.(*http.Handler).Handler h.ServeHTTP(w, req) @@ -688,9 +873,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("POST", "/index/idx1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"index already exists"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // create field @@ -708,9 +893,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("POST", "/index/idx1/field/fld1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"field already exists"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // delete field @@ -728,9 +913,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("DELETE", "/index/idx1/field/fld1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"field not found"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"deleting field: field not found"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } // delete index @@ -748,9 +933,9 @@ func TestHandler_Endpoints(t *testing.T) { r = test.MustNewHTTPRequest("DELETE", "/index/idx1", strings.NewReader("")) h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"index not found"}}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + t.Errorf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { + t.Errorf("unexpected body: %q", w.Body.String()) } }) @@ -844,7 +1029,10 @@ func TestClusterTranslator(t *testing.T) { cluster := make(test.Cluster, 2) cluster[0] = test.NewCommandNode(true) cluster[0].Config.Gossip.Port = "0" - cluster[0].Start() + err := cluster[0].Start() + if err != nil { + t.Fatalf("starting cluster 1: %v", err) + } httpTranslateStore := http.NewTranslateStore(cluster[0].URL()) cluster[1] = test.NewCommandNode(false, server.OptCommandServerOptions( @@ -853,7 +1041,10 @@ func TestClusterTranslator(t *testing.T) { ) cluster[1].Config.Gossip.Port = "0" cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()} - cluster[1].Start() + err = cluster[1].Start() + if err != nil { + t.Fatalf("starting cluster 1: %v", err) + } test.MustDo("POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}") test.MustDo("POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}") @@ -907,3 +1098,20 @@ func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) { } return ret } + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +func (r getSchemaResponse) findField(indexName, fieldName string) *pilosa.FieldInfo { + for _, index := range r.Indexes { + if index.Name == indexName { + for _, field := range index.Fields { + if field.Name == fieldName { + return field + } + } + } + } + return nil +} diff --git a/server/server_test.go b/server/server_test.go index 1cc3bd87f..11a026dea 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -321,6 +321,7 @@ func TestConfig_Parse_DataDir(t *testing.T) { func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 cluster := test.MustRunCluster(t, clusterSize) + defer cluster.Close() // Create the schema. client0 := cluster[0].Client() @@ -929,7 +930,10 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { bm := roaring.NewBitmap() bm.DirectAdd(0) buf := &bytes.Buffer{} - bm.WriteTo(buf) + _, err := bm.WriteTo(buf) + if err != nil { + t.Fatalf("writing to buffer: %v", err) + } data := buf.Bytes() eg := errgroup.Group{} @@ -952,7 +956,7 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { return nil }) } - err := eg.Wait() + err = eg.Wait() if err != nil { t.Fatalf("setting lots of shards: %v", err) } diff --git a/server/setup_logger.go b/server/setup_logger.go index 06d61eced..3473faa43 100644 --- a/server/setup_logger.go +++ b/server/setup_logger.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !arm64 package server diff --git a/server/setup_logger_arm64.go b/server/setup_logger_arm64.go index dd6718741..090ac1395 100644 --- a/server/setup_logger_arm64.go +++ b/server/setup_logger_arm64.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( diff --git a/server_internal_test.go b/server_internal_test.go index 86688ebbe..a8af8186c 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -16,12 +16,9 @@ package pilosa import ( "io/ioutil" - "os" "runtime" "testing" "time" - - "github.com/pilosa/pilosa/roaring" ) // Ensure the file handle count is working @@ -63,58 +60,3 @@ func TestMonitorAntiEntropyZero(t *testing.T) { t.Fatalf("monitorAntiEntropy should have returned immediately with duration 0") } } - -func TestOnlyOpenOwnedFiles(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa") - if err != nil { - t.Fatalf("getting temp dir: %v", err) - } - defer func() { - err := os.RemoveAll(path) - if err != nil { - t.Logf("cleaning up temp dir: %v", err) - } - }() - bm := roaring.NewFileBitmap(1, 2, 3) - err = os.MkdirAll(path+"/i/f/views/standard/fragments", os.ModeDir|os.ModePerm) - if err != nil { - t.Fatalf("mkdirall: %v", err) - } - one, err := os.Create(path + "/i/f/views/standard/fragments/1") - if err != nil { - t.Fatalf("creating one: %v", err) - } - two, err := os.Create(path + "/i/f/views/standard/fragments/2") - if err != nil { - t.Fatalf("creating two: %v", err) - } - _, err = bm.WriteTo(one) - if err != nil { - t.Fatalf("writing to one: %v", err) - } - _, err = bm.WriteTo(two) - if err != nil { - t.Fatalf("writing to two: %v", err) - } - - h := NewHolder() - h.Path = path - h.shardValidatorFunc = func(index string, shard uint64) bool { - return shard == 1 - } - - err = h.Open() - if err != nil { - t.Fatalf("opening holder: %v", err) - } - - view := h.Index("i").Field("f").view("standard") - - if len(view.fragments) != 1 { - t.Errorf("should have one fragment, but have: %d", len(view.fragments)) - } - - if _, ok := view.fragments[1]; !ok { - t.Errorf("should have fragment 1, but fragments: %#v", view.fragments) - } -} diff --git a/shardwidth/16.go b/shardwidth/16.go new file mode 100644 index 000000000..6334bf254 --- /dev/null +++ b/shardwidth/16.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth16 + +package shardwidth + +const Exponent = 16 diff --git a/shardwidth/17.go b/shardwidth/17.go new file mode 100644 index 000000000..b6ba8d907 --- /dev/null +++ b/shardwidth/17.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth17 + +package shardwidth + +const Exponent = 17 diff --git a/shardwidth/18.go b/shardwidth/18.go new file mode 100644 index 000000000..f69647681 --- /dev/null +++ b/shardwidth/18.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth18 + +package shardwidth + +const Exponent = 18 diff --git a/shardwidth/19.go b/shardwidth/19.go new file mode 100644 index 000000000..e9920326b --- /dev/null +++ b/shardwidth/19.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth19 + +package shardwidth + +const Exponent = 19 diff --git a/shardwidth/20.go b/shardwidth/20.go new file mode 100644 index 000000000..5505656dd --- /dev/null +++ b/shardwidth/20.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 + +package shardwidth + +const Exponent = 20 diff --git a/shardwidth/21.go b/shardwidth/21.go new file mode 100644 index 000000000..0ab1be104 --- /dev/null +++ b/shardwidth/21.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth21 + +package shardwidth + +const Exponent = 21 diff --git a/shardwidth/22.go b/shardwidth/22.go new file mode 100644 index 000000000..9b07fff50 --- /dev/null +++ b/shardwidth/22.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth22 + +package shardwidth + +const Exponent = 22 diff --git a/shardwidth/23.go b/shardwidth/23.go new file mode 100644 index 000000000..03c5cd91c --- /dev/null +++ b/shardwidth/23.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth23 + +package shardwidth + +const Exponent = 23 diff --git a/shardwidth/24.go b/shardwidth/24.go new file mode 100644 index 000000000..7e175b1bc --- /dev/null +++ b/shardwidth/24.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth24 + +package shardwidth + +const Exponent = 24 diff --git a/shardwidth/25.go b/shardwidth/25.go new file mode 100644 index 000000000..776655580 --- /dev/null +++ b/shardwidth/25.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth25 + +package shardwidth + +const Exponent = 25 diff --git a/shardwidth/26.go b/shardwidth/26.go new file mode 100644 index 000000000..1459686bc --- /dev/null +++ b/shardwidth/26.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth26 + +package shardwidth + +const Exponent = 26 diff --git a/shardwidth/27.go b/shardwidth/27.go new file mode 100644 index 000000000..9eb7ff830 --- /dev/null +++ b/shardwidth/27.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth27 + +package shardwidth + +const Exponent = 27 diff --git a/shardwidth/28.go b/shardwidth/28.go new file mode 100644 index 000000000..b3c38c944 --- /dev/null +++ b/shardwidth/28.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth28 + +package shardwidth + +const Exponent = 28 diff --git a/shardwidth/29.go b/shardwidth/29.go new file mode 100644 index 000000000..e2b29faf1 --- /dev/null +++ b/shardwidth/29.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth29 + +package shardwidth + +const Exponent = 29 diff --git a/shardwidth/30.go b/shardwidth/30.go new file mode 100644 index 000000000..a8684c236 --- /dev/null +++ b/shardwidth/30.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth30 + +package shardwidth + +const Exponent = 30 diff --git a/shardwidth/31.go b/shardwidth/31.go new file mode 100644 index 000000000..1dcd40ebb --- /dev/null +++ b/shardwidth/31.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth31 + +package shardwidth + +const Exponent = 31 diff --git a/shardwidth/32.go b/shardwidth/32.go new file mode 100644 index 000000000..d9408285b --- /dev/null +++ b/shardwidth/32.go @@ -0,0 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build shardwidth32 + +package shardwidth + +const Exponent = 32 diff --git a/stats/stats_test.go b/stats/stats_test.go index 3da83ce70..f09cce79e 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -211,7 +211,9 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } func TestStatsCount_APICalls(t *testing.T) { - cmd := test.MustRunCluster(t, 1)[0] + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/syswrap/mmap.go b/syswrap/mmap.go index a13dff8af..95e819999 100644 --- a/syswrap/mmap.go +++ b/syswrap/mmap.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package syswrap wraps syscalls (just mmap right now) in order to impose a // global in-process limit on the maximum number of active mmaps. package syswrap diff --git a/syswrap/os.go b/syswrap/os.go index 8916d593b..1704b0759 100644 --- a/syswrap/os.go +++ b/syswrap/os.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package syswrap import ( diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 1f00ff68b..1d0feb237 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -28,6 +28,7 @@ import ( func TestNewCluster(t *testing.T) { numNodes := 3 cluster := test.MustRunCluster(t, numNodes) + defer cluster.Close() coordinator := getCoordinator(cluster[0]) for i := 1; i < numNodes; i++ { diff --git a/toml/toml.go b/toml/toml.go index 5193ad787..acfb93079 100644 --- a/toml/toml.go +++ b/toml/toml.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package toml import "time" diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index aacc42d6e..47ec439fc 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package opentracing import ( diff --git a/tracing/tracing.go b/tracing/tracing.go index 5792dd7ca..3625166b0 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tracing import ( diff --git a/translate.go b/translate.go index db129cd8f..94eb57186 100644 --- a/translate.go +++ b/translate.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -19,6 +33,7 @@ import ( "github.com/pkg/errors" ) +// Log entry type constants. const ( LogEntryTypeInsertColumn = 1 LogEntryTypeInsertRow = 2 @@ -28,6 +43,7 @@ const ( defaultReplicationRetryInterval = 1 * time.Second ) +// Translate store errors. var ( ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") @@ -85,12 +101,17 @@ type TranslateFile struct { // TranslateFileOption is a functional option type for pilosa.TranslateFile type TranslateFileOption func(f *TranslateFile) error +// OptTranslateFileMapSize is a functional option on TranslateFile +// used to set the map size. func OptTranslateFileMapSize(mapSize int) TranslateFileOption { return func(f *TranslateFile) error { f.mapSize = mapSize return nil } } + +// OptTranslateFileLogger is a functional option on TranslateFile +// used to set the file logger. func OptTranslateFileLogger(l logger.Logger) TranslateFileOption { return func(s *TranslateFile) error { s.logger = l @@ -137,6 +158,7 @@ func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { return f } +// Open opens the translate file. func (s *TranslateFile) Open() (err error) { // Open writer & buffered writer. if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { @@ -145,6 +167,7 @@ func (s *TranslateFile) Open() (err error) { return errors.Wrapf(err, "open file %s", s.Path) } s.w = bufio.NewWriter(s.file) + s.n = 0 // Memory map data file. if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { @@ -218,6 +241,7 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error { return nil } +// Close closes the translate file. func (s *TranslateFile) Close() (err error) { s.once.Do(func() { close(s.closing) @@ -574,6 +598,7 @@ func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (str return "", nil } +// TranslateRowsToUint64 converts a slice of row keys to a slice of row IDs. func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) { key := fieldKey{index, field} @@ -665,6 +690,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []stri return ret, nil } +// TranslateRowToString translates a row ID to a string key. func (s *TranslateFile) TranslateRowToString(index, field string, id uint64) (string, error) { s.mu.RLock() if idx := s.rows[fieldKey{index, field}]; idx != nil { @@ -686,6 +712,8 @@ func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser return rc, nil } +// LogEntry is a batch of Key/ID mappings which is replicated to other nodes +// for read-only key translation. type LogEntry struct { Type uint8 Index []byte diff --git a/translate_test.go b/translate_test.go index 2b5d35a49..4fa977e62 100644 --- a/translate_test.go +++ b/translate_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -744,6 +758,156 @@ func TestTranslateFile_ReassignPrimaryTranslateStore(t *testing.T) { }) } +func TestTranslateFile_ReopenTheSameInstance(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + if value, err := s.TranslateColumnToString("IDX0", 1); err != nil { + t.Fatal(err) + } else if value != "foo" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + // Reopen the store. + s.MustOpen() + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } +} + func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { const batchSize = 1000 @@ -822,10 +986,14 @@ func (t *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser func MustOpenTranslateFile() *TranslateFile { s := NewTranslateFile() + s.MustOpen() + return s +} + +func (s *TranslateFile) MustOpen() { if err := s.Open(); err != nil { panic(err) } - return s } func (s *TranslateFile) Close() error { diff --git a/uri.go b/uri.go index 231457691..00216afbd 100644 --- a/uri.go +++ b/uri.go @@ -56,8 +56,11 @@ func defaultURI() *URI { } } +// URIs is a convenience type representing a slice of URI. type URIs []URI +// HostPortStrings returns a slice of host:port strings +// based on the slice of URI. func (u URIs) HostPortStrings() []string { s := make([]string, len(u)) for i, a := range u { @@ -199,6 +202,7 @@ func (u *URI) MarshalJSON() ([]byte, error) { return json.Marshal(output) } +// UnmarshalJSON unmarshals a byte slice to a URI. func (u *URI) UnmarshalJSON(b []byte) error { var input struct { Scheme string `json:"scheme,omitempty"` diff --git a/utils_internal_test.go b/utils_internal_test.go index bf98f08a8..76e70fb9e 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -57,15 +57,15 @@ func NewTestCluster(n int) *cluster { // NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { uri := defaultURI() - uri.setScheme(scheme) - uri.setHost(host) + _ = uri.setScheme(scheme) + _ = uri.setHost(host) uri.SetPort(port) return *uri } func NewTestURIFromHostPort(host string, port uint16) URI { uri := defaultURI() - uri.setHost(host) + _ = uri.setHost(host) uri.SetPort(port) return *uri } @@ -235,7 +235,9 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.addNode(n) + if err := c.addNode(n); err != nil { + return nil, err + } } } @@ -314,7 +316,10 @@ func (b bcast) SendSync(m Message) error { // Apply the send message to all nodes (except the coordinator). for _, c := range b.t.Clusters { if c != b.c { - c.mergeClusterStatus(obj) + err := c.mergeClusterStatus(obj) + if err != nil { + return err + } } } b.t.mu.RLock() @@ -348,12 +353,17 @@ func (b bcast) SendTo(to *Node, m Message) error { } case *ResizeInstructionComplete: coord := b.t.clusterByID(to.ID) - go coord.markResizeInstructionComplete(obj) + // this used to be async, but that prevented us from checking + // its error status... + return coord.markResizeInstructionComplete(obj) case *ClusterStatus: // Apply the send message to the node. for _, c := range b.t.Clusters { if c.Node.ID == to.ID { - c.mergeClusterStatus(obj) + err := c.mergeClusterStatus(obj) + if err != nil { + return err + } } } b.t.mu.RLock() diff --git a/view.go b/view.go index 5780e682c..bd917f407 100644 --- a/view.go +++ b/view.go @@ -52,11 +52,10 @@ type view struct { // Fragments by shard. fragments map[uint64]*fragment - broadcaster broadcaster - stats stats.StatsClient - rowAttrStore AttrStore - logger logger.Logger - shardValidator func(uint64) bool + broadcaster broadcaster + stats stats.StatsClient + rowAttrStore AttrStore + logger logger.Logger } // newView returns a new instance of View. @@ -73,10 +72,9 @@ func newView(path, index, field, name string, fieldOptions FieldOptions) *view { fragments: make(map[uint64]*fragment), - broadcaster: NopBroadcaster, - stats: stats.NopStatsClient, - logger: logger.NopLogger, - shardValidator: defaultShardValidator, + broadcaster: NopBroadcaster, + stats: stats.NopStatsClient, + logger: logger.NopLogger, } } @@ -90,12 +88,14 @@ func (v *view) open() error { if err := func() error { // Ensure the view's path exists. + v.logger.Debugf("ensure view path exists: %s", v.path) if err := os.MkdirAll(v.path, 0777); err != nil { return errors.Wrap(err, "creating view directory") } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { return errors.Wrap(err, "creating fragments directory") } + v.logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name) if err := v.openFragments(); err != nil { return errors.Wrap(err, "opening fragments") } @@ -106,6 +106,7 @@ func (v *view) open() error { return err } + v.logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name) return nil } @@ -132,18 +133,17 @@ func (v *view) openFragments() error { // Parse filename into integer. shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { - continue - } - //skip shard if not owned - if !v.shardValidator(shard) { + v.logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name()) continue } + v.logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard) frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } frag.RowAttrStore = v.rowAttrStore + v.logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard) v.fragments[frag.shard] = frag } @@ -166,6 +166,15 @@ func (v *view) close() error { return nil } +// flags returns a set of flags for the underlying fragments. +func (v *view) flags() byte { + var flag byte + if v.fieldType == FieldTypeInt { + flag |= roaringFlagBSIv2 + } + return flag +} + // availableShards returns a bitmap of shards which contain data. func (v *view) availableShards() *roaring.Bitmap { v.mu.RLock() @@ -173,7 +182,7 @@ func (v *view) availableShards() *roaring.Bitmap { b := roaring.NewBitmap() for shard := range v.fragments { - b.Add(shard) // ignore error, no writer attached + _, _ = b.Add(shard) // ignore error, no writer attached } return b } @@ -254,7 +263,7 @@ func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { } func (v *view) newFragment(path string, shard uint64) *fragment { - frag := newFragment(path, v.index, v.field, v.name, shard) + frag := newFragment(path, v.index, v.field, v.name, shard, v.flags()) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.logger @@ -331,7 +340,7 @@ func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) { } // value uses a column of bits to read a multi-bit value. -func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +func (v *view) value(columnID uint64, bitDepth uint) (value int64, exists bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { @@ -341,7 +350,7 @@ func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, } // setValue uses a column of bits to set a multi-bit value. -func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { @@ -351,7 +360,7 @@ func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed b } // sum returns the sum & count of a field. -func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (v *view) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { for _, f := range v.allFragments() { fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { @@ -364,7 +373,7 @@ func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { } // min returns the min and count of a field. -func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) { +func (v *view) min(filter *Row, bitDepth uint) (min int64, count uint64, err error) { var minHasValue bool for _, f := range v.allFragments() { fmin, fcount, err := f.min(filter, bitDepth) @@ -392,7 +401,7 @@ func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) { } // max returns the max and count of a field. -func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (v *view) max(filter *Row, bitDepth uint) (max int64, count uint64, err error) { for _, f := range v.allFragments() { fmax, fcount, err := f.max(filter, bitDepth) if err != nil { @@ -407,7 +416,7 @@ func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) { } // rangeOp returns rows with a field value encoding matching the predicate. -func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) { r := NewRow() for _, frag := range v.allFragments() { other, err := frag.rangeOp(op, bitDepth, predicate) @@ -419,6 +428,28 @@ func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, err return r, nil } +// upgradeViewBSIv2 upgrades the fragments of v. Returns ok true if any fragment upgraded. +func upgradeViewBSIv2(v *view, bitDepth uint) (ok bool, _ error) { + // If reading from an old formatted BSI roaring bitmap, upgrade and reload. + for _, frag := range v.allFragments() { + if frag.storage.Flags&roaringFlagBSIv2 == 1 { + continue // already upgraded, skip + } + ok = true // mark as upgraded, requires reload + + if tmpPath, err := upgradeRoaringBSIv2(frag, bitDepth); err != nil { + return ok, errors.Wrap(err, "upgrading bsi v2") + } else if err := frag.closeStorage(); err != nil { + return ok, errors.Wrap(err, "closing after bsi v2 upgrade") + } else if err := os.Rename(tmpPath, frag.path); err != nil { + return ok, errors.Wrap(err, "renaming after bsi v2 upgrade") + } else if err := frag.openStorage(); err != nil { + return ok, errors.Wrap(err, "re-opening after bsi v2 upgrade") + } + } + return ok, nil +} + // ViewInfo represents schema information for a view. type ViewInfo struct { Name string `json:"name"`