diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ba2868e6..a27e34be1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ fast-checkout: &fast-checkout attach_workspace: at: . jobs: - build: + setup: <<: *defaults steps: - checkout @@ -30,6 +30,14 @@ jobs: - run: gometalinter --install - run: go get github.com/remyoudompheng/go-misc/deadcode - run: make gometalinter + test-build-arm: + <<: *defaults + steps: + - *fast-checkout + - run: make build GOOS=linux GOARCH=arm GOARM=5 + - run: make build GOOS=linux GOARCH=arm GOARM=6 + - run: make build GOOS=linux GOARCH=arm GOARM=7 + - run: make build GOOS=linux GOARCH=arm64 test-golang-1.11: &base-test <<: *defaults steps: @@ -42,7 +50,7 @@ jobs: - *fast-checkout - run: sudo apt-get install lsof - run: - command: make test TESTFLAGS="-race -timeout=30m" + command: make test TESTFLAGS="-race -v -timeout=30m" no_output_timeout: 30m test-golang-1.11-386: <<: *base-test @@ -100,25 +108,28 @@ workflows: version: 2 test: jobs: - - build + - setup - linter: requires: - - build + - setup + - test-build-arm: + requires: + - setup - test-golang-1.11: requires: - - build + - setup - test-golang-1.11-race: requires: - - build + - setup - test-golang-1.11-386: requires: - - build + - setup - test-golang-1.10: requires: - - build + - setup - cluster-tests: requires: - - build + - setup - prerelease: requires: - linter diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 1dce0ad40..05043a418 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,8 +1,8 @@ For bugs, please provide the following: -### Expected behavior +### What's going wrong? -### Actual behavior +### What was expected? ### Steps to reproduce the behavior diff --git a/CHANGELOG.md b/CHANGELOG.md index ca283b2c7..b7b558ac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,96 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [v1.1.0] - 2018-08-21 +## [1.2.0] - 2018-12-20 + +This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. + +### Added + +- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) +- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) +- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) +- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) +- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) +- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) +- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) +- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) +- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) +- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) +- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) +- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) +- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) +- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) +- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) +- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) +- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) +- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) +- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) +- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) +- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) +- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) +- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) +- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) +- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) +- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) +- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) +- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) + +### Changed + +- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) +- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) +- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) +- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) +- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) +- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) +- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) +- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) + +### Performance + +- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) +- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) + +### Fixed + +- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) +- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) +- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) +- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) +- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) +- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) +- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) +- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) +- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) +- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) +- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) +- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) +- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) +- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) +- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) +- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) +- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) +- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) +- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) +- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) +- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) +- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) +- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) +- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) +- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) +- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) +- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) +- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) +- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) +- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) +- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) +- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) +- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) +- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) + + +## [1.1.0] - 2018-08-21 This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. @@ -29,7 +118,7 @@ This version contains 32 contributions from 5 contributors. There are 89 files c - Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) - Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) -## [v1.0.2] - 2018-08-01 +## [1.0.2] - 2018-08-01 This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. @@ -44,7 +133,7 @@ This version contains 11 contributions from 3 contributors. There are 30 files c - Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) - Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) -## [v1.0.1] - 2018-07-11 +## [1.0.1] - 2018-07-11 This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. @@ -56,7 +145,7 @@ This version contains 12 contributions from 4 contributors. There are 11 files c - Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) - Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) -## [v1.0.0] - 2018-07-09 +## [1.0.0] - 2018-07-09 This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. @@ -104,7 +193,7 @@ This version contains 218 contributions from 7 contributors. There are 184 files - Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) - Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) -## [v0.10.0] - 2018-05-15 +## [0.10.0] - 2018-05-15 This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. @@ -135,7 +224,7 @@ This version contains 93 contributions from 8 contributors. There are 93 files c - Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) - Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) -## [v0.9.0] - 2018-05-04 +## [0.9.0] - 2018-05-04 This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. @@ -489,7 +578,7 @@ This version contains 53 contributions from 13 contributors (including 4 volunte - Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) - Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) -[Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD +[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD [0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 [0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 [0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 @@ -499,3 +588,4 @@ This version contains 53 contributions from 13 contributors (including 4 volunte [0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 [1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 [1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 +[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/Dockerfile b/Dockerfile index e4f057564..02454a9c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 as builder +FROM golang:1.11.4 as builder COPY . /go/src/github.com/pilosa/pilosa/ diff --git a/Makefile b/Makefile index af86b8241..e7af8e119 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ clean: rm -rf vendor build # Set up vendor directory using `dep` -vendor: Gopkg.toml +vendor: Gopkg.toml Gopkg.lock $(MAKE) require-dep dep ensure -vendor-only touch vendor @@ -68,20 +68,24 @@ release: check-clean $(MAKE) release-build GOOS=linux GOARCH=386 $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 + +# try (e.g.) internal/clustertests/docker-compose-replication2.yml +DOCKER_COMPOSE=internal/clustertests/docker-compose.yml + # Run cluster integration tests using docker. Requires docker daemon to be # running. This will catch changes to internal/clustertests/*.go, but if you # make changes to Pilosa, you'll want to run clustertests-build to rebuild the # pilosa image. clustertests: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml build client1 - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) build client1 + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 # Like clustertests, but rebuilds all images. clustertests-build: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build # Create prerelease builds prerelease: vendor diff --git a/api.go b/api.go index 92ac7672e..d3fb03615 100644 --- a/api.go +++ b/api.go @@ -333,8 +333,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } return err }) - go func(node *Node) { - }(node) } else if !remote { // if remote == true we don't forward to other nodes // forward it on eg.Go(func() error { @@ -561,6 +559,23 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa return blocks, nil } +// FragmentData returns all data in the specified fragment. +func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentData") + defer span.Finish() + + if err := api.validate(apiFragmentData); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Retrieve fragment from holder. + f := api.holder.fragment(indexName, fieldName, viewName, shard) + if f == nil { + return nil, ErrFragmentNotFound + } + return f, nil +} + // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. func (api *API) Hosts(ctx context.Context) []*Node { @@ -1157,6 +1172,36 @@ func (api *API) Info() serverInfo { } } +func (api *API) TranslateKeys(body io.Reader) ([]byte, error) { + reqBytes, err := ioutil.ReadAll(body) + if err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "read body error")) + } + var req TranslateKeysRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) + } + var ids []uint64 + if req.Field == "" { + ids, err = api.holder.translateFile.TranslateColumnsToUint64(req.Index, req.Keys) + } else { + ids, err = api.holder.translateFile.TranslateRowsToUint64(req.Index, req.Field, req.Keys) + } + if err != nil { + return nil, err + } + + resp := TranslateKeysResponse{ + IDs: ids, + } + // Encode response. + buf, err := api.Serializer.Marshal(&resp) + if err != nil { + return nil, errors.Wrap(err, "translate keys response encoding error") + } + return buf, nil +} + type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` } @@ -1175,6 +1220,7 @@ const ( apiExportCSV apiFragmentBlockData apiFragmentBlocks + apiFragmentData apiField apiFieldAttrDiff //apiHosts // not implemented @@ -1204,7 +1250,8 @@ var methodsCommon = map[apiMethod]struct{}{ } var methodsResizing = map[apiMethod]struct{}{ - apiResizeAbort: {}, + apiFragmentData: {}, + apiResizeAbort: {}, } var methodsNormal = map[apiMethod]struct{}{ diff --git a/api_test.go b/api_test.go index e569896f0..a33b064c1 100644 --- a/api_test.go +++ b/api_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -48,7 +49,7 @@ func TestAPI_Import(t *testing.T) { index := "rick" field := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) + _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } @@ -99,6 +100,22 @@ func TestAPI_Import(t *testing.T) { } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %+v", keys) } + + }) + + // Relies on the previous test creating an index with TrackExistence and + // adding some data. + t.Run("SchemaHasNoExists", func(t *testing.T) { + schema := m1.API.Schema(context.Background()) + for _, f := range schema[0].Fields { + if f.Name == "_exists" { + t.Fatalf("found _exists field in schema") + } + if strings.HasPrefix(f.Name, "_") { + t.Fatalf("found internal field '%s' in schema output", f.Name) + } + } + }) t.Run("RowKeyColumnID", func(t *testing.T) { @@ -206,7 +223,7 @@ func TestAPI_ImportValue(t *testing.T) { t.Fatal(err) } - pql := fmt.Sprintf("Range(%s>0)", field) + pql := fmt.Sprintf("Row(%s>0)", field) // Query node0. if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { diff --git a/client.go b/client.go index b16222b7a..3762a27b9 100644 --- a/client.go +++ b/client.go @@ -52,7 +52,7 @@ type InternalClient interface { ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) + RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error } @@ -149,6 +149,6 @@ func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fie func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { +func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index 451ce8ba1..e966c57a1 100644 --- a/cluster.go +++ b/cluster.go @@ -70,7 +70,7 @@ type Node struct { } func (n Node) String() string { - return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6]) + return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID) } // Nodes represents a list of nodes. @@ -364,6 +364,7 @@ func (c *cluster) addNode(node *Node) error { if !c.Topology.addID(node.ID) { return nil } + c.Topology.nodeStates[node.ID] = node.State // save topology return c.saveTopology() @@ -588,6 +589,12 @@ func (c *cluster) nodePositionByID(nodeID string) int { func (c *cluster) addNodeBasicSorted(node *Node) bool { n := c.unprotectedNodeByID(node.ID) if n != nil { + if n.State != node.State || n.IsCoordinator != node.IsCoordinator || n.URI != node.URI { + n.State = node.State + n.IsCoordinator = node.IsCoordinator + n.URI = node.URI + return true + } return false } @@ -829,6 +836,13 @@ func (c *cluster) partition(index string, shard uint64) int { return int(h.Sum64() % uint64(c.partitionN)) } +// ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. +func (c *cluster) ShardNodes(index string, shard uint64) []*Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.shardNodes(index, shard) +} + // shardNodes returns a list of nodes that own a fragment. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { return c.partitionNodes(c.partition(index, shard)) @@ -976,7 +990,7 @@ func (c *cluster) markAsJoined() { // needTopologyAgreement is unprotected. func (c *cluster) needTopologyAgreement() bool { - return c.state == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) + return (c.state == ClusterStateStarting || c.state == ClusterStateDegraded) && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } // haveTopologyAgreement is unprotected. @@ -1204,7 +1218,7 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (* Node: toCluster.unprotectedNodeByID(id), Coordinator: c.unprotectedCoordinatorNode(), Sources: sources, - Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. + NodeStatus: c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node. ClusterStatus: c.unprotectedStatus(), } j.Instructions = append(j.Instructions, instr) @@ -1263,12 +1277,30 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { span, ctx := tracing.StartSpanFromContext(context.Background(), "Cluster.followResizeInstruction") defer span.Finish() - // Sync the schema received in the resize instruction. + // Sync the NodeStatus received in the resize instruction. + // Sync schema. c.logger.Debugf("holder applySchema") - if err := c.holder.applySchema(instr.Schema); err != nil { + if err := c.holder.applySchema(instr.NodeStatus.Schema); err != nil { return errors.Wrap(err, "applying schema") } + // Sync available shards. + for _, is := range instr.NodeStatus.Indexes { + for _, fs := range is.Fields { + f := c.holder.Field(is.Name, fs.Name) + + // if we don't know about a field locally, log an error because + // fields should be created and synced prior to shard creation + if f == nil { + c.logger.Printf("local field not found: %s/%s", is.Name, fs.Name) + continue + } + if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } + } + } + // Request each source file in ResizeSources. for _, src := range instr.Sources { c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) @@ -1295,7 +1327,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // Stream shard from remote node. c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.Shard, srcURI) + rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.View, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a shard has been skipped and @@ -1304,7 +1336,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // TODO: figure out a way to distinguish from "fragment not found" errors // which are true errors and which simply mean the fragment doesn't have data. if err == ErrFragmentNotFound { - return nil + continue } return errors.Wrap(err, "retrieving shard") } else if rd == nil { @@ -1803,6 +1835,30 @@ func (c *cluster) nodeLeave(nodeID string) error { return nil } +func (c *cluster) nodeStatus() *NodeStatus { + ns := &NodeStatus{ + Node: c.Node, + Schema: &Schema{Indexes: c.holder.Schema()}, + } + var availableShards *roaring.Bitmap + for _, idx := range ns.Schema.Indexes { + is := &IndexStatus{Name: idx.Name} + for _, f := range idx.Fields { + if field := c.holder.Field(idx.Name, f.Name); field != nil { + availableShards = field.AvailableShards() + } else { + availableShards = roaring.NewBitmap() + } + is.Fields = append(is.Fields, &FieldStatus{ + Name: f.Name, + AvailableShards: availableShards, + }) + } + ns.Indexes = append(ns.Indexes, is) + } + return ns +} + func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() @@ -1904,7 +1960,7 @@ type ResizeInstruction struct { Node *Node Coordinator *Node Sources []*ResizeSource - Schema *Schema + NodeStatus *NodeStatus ClusterStatus *ClusterStatus } diff --git a/ctl/import_test.go b/ctl/import_test.go index ad4fd6cf0..e701af1b2 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -18,9 +18,11 @@ import ( "bufio" "bytes" "context" + "fmt" "io" "io/ioutil" "net/http" + "reflect" "strings" "testing" @@ -184,6 +186,69 @@ func TestImportCommand_RunKeys(t *testing.T) { } } +// Ensure that import with keys runs with key replication. +func TestImportCommand_KeyReplication(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + file, err := ioutil.TempFile("", "import-key.csv") + + // create a large import file in order to test the + // translateStoreBufferSize growth logic. + keyBytes := []byte{} + for row := 0; row < 100; row++ { + for col := 0; col < 100; col++ { + x := fmt.Sprintf("foo%d,bar%d\n", row, col) + keyBytes = append(keyBytes, x...) + } + } + x := "fooEND,barEND" + keyBytes = append(keyBytes, x...) + + file.Write(keyBytes) + ctx := context.Background() + if err != nil { + t.Fatal(err) + } + + c := test.MustRunCluster(t, 2) + cmd0 := c[0] + cmd1 := c[1] + + host0 := cmd0.API.Node().URI.HostPort() + host1 := cmd1.API.Node().URI.HostPort() + + cm.Host = host0 + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + + cm.Index = "i" + cm.Field = "f" + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with key replication doesn't work: %s", err) + } + + // Verify that the data is available on both nodes. + for _, host := range []string{host0, host1} { + qry := "Count(Row(f=foo0))" + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry))) + if err != nil { + t.Fatalf("Querying data for validation: %s", err) + } + + // Read body and unmarshal response. + exp := `{"results":[100]}` + "\n" + if body, err := ioutil.ReadAll(resp.Body); err != nil { + t.Fatalf("reading: %s", err) + } else if !reflect.DeepEqual(body, []byte(exp)) { + t.Fatalf("expected: %s, but got: %s", exp, body) + } + } +} + // Ensure that integer import with keys runs. func TestImportCommand_RunValueKeys(t *testing.T) { buf := bytes.Buffer{} diff --git a/docs/administration.md b/docs/administration.md index 3f07a5b68..4bfa5470a 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -323,7 +323,7 @@ We currently track the following events - **Xor:** Count of Xor queries. - **Not:** Count of Not queries. - **Count:** Count of Count queries. -- **Range:** Count of Range queries. +- **Range:** Count of ranged Row queries. - **Snapshot:** Event count when the snapshot process is triggered. - **BlockRepair:** Count of data blocks that were out of sync and repaired. - **GarbageCollection:** Event count when garbage collection occurs. diff --git a/docs/data-model.md b/docs/data-model.md index 9bb44bb99..1a0781ccc 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -64,7 +64,7 @@ Simple queries: Relational | Pilosa -----------------------------------------------|------------------------------------ `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` - `select ID from People where Age > 30` | `Range(Age > 30)` + `select ID from People where Age > 30` | `Row(Age > 30)` `select ID from People where Member = true` | `Row(Member=0)` Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: @@ -100,7 +100,7 @@ The LRU cache maintains the most recently accessed Rows. ### Time Quantum -Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example, if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to `YMD`, ranged Row queries down to the granularity of a day are supported. ### Attribute @@ -145,7 +145,7 @@ curl localhost:10101/index/repository/field/quantity \ ##### BSI Range-Encoding -Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. +Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Row`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. diff --git a/docs/glossary.md b/docs/glossary.md index 86ae6bd70..9ff232454 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -46,16 +46,16 @@ nav = [] [Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -[Range](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). - -[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). - [Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. [Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. [Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap). +[Row (Ranged)](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). + +[Row (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). + [Slice](../data-model/#shard): Prior to Pilosa 1.0, shards were known as slices. [Shard](../data-model/#shard): [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). @@ -64,7 +64,7 @@ nav = [] [Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field). -[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [Range](#range) queries on time [fields](#field). +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [ranged Row](#range) queries on time [fields](#field). [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). diff --git a/docs/installation.md b/docs/installation.md index b2e94d1a0..9bd6f9bef 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.1.0-darwin-amd64.tar.gz + tar xfz pilosa-v1.2.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.1.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.2.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -222,19 +222,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.1.0-linux-amd64.tar.gz + tar xfz pilosa-v1.2.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.1.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.2.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -251,7 +251,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -314,7 +314,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: diff --git a/docs/query-language.md b/docs/query-language.md index 0f8feb795..73e1cab7d 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -50,7 +50,7 @@ curl localhost:10101/index/repository/query \ * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. * `CALL` Any query -* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`, `Not` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -335,6 +335,88 @@ Row(stargazer=1) * attrs are the attributes for user 1 * columns are the repositories which user 1 has starred. + +#### Row (Range) + +**Spec:** + +``` +Row(=, from=, to=) +``` + +**Description:** + +Similar to `Row`, but only returns bits which were set with timestamps between the given `from` (inclusive) and `to` (exclusive) timestamps. Both `from` and `to` parameters are optional. The default for `to` timestamp is current time + 1 day. If a later end timestamp is required, specify it explicitly. + +**Result Type:** object with attrs and bits + + +**Examples:** + +Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: +```request +Row(stargazer=1, from='2010-01-01T00:00', to='2017-03-02T03:00') +``` +```response +{{"attrs":{},"columns":[10]} +``` + +This example assumes timestamps have been set on some bits. + +* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. + + +#### Row (BSI) + +**Spec:** + +``` +Row([ ] ) +``` + +**Description:** + +The `Row` query is overloaded to work on `integer` values as well as `timestamp` values. +Returns bits that are true for the comparison operator. + +**Result Type:** object with attrs and columns + +**Examples:** + +In our source data, commitactivity was counted over the last year. +The following greater-than `Row` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): + +```request +Row(commitactivity > 100) +``` +```response +{{"attrs":{},"columns":[10]} +``` + +* columns are repositories which had at least 100 commits in the last year. + +BSI range queries support the following operators: + + Operator | Name | Value +----------|-------------------------------|-------------------- + `>` | greater-than, GT | integer + `<` | less-than, LT | integer + `<=` | less-than-or-equal-to, LTE | integer + `>=` | greater-than-or-equal-to, GTE | integer + `==` | equal-to, EQ | integer + `!=` | not-equal-to, NEQ | integer or `null` + +`<`, and `<=` can be chained together to represent a bounded interval. For example: + +```request +Row(50 < commitactivity < 150) +``` +```response +{{"attrs":{},"columns":[10]} +``` + +As of Pilosa 1.0, the "between" syntax `Row(frame=stats, commitactivity >< [50, 150])` is no longer supported. + #### Union **Spec:** @@ -584,87 +666,6 @@ TopN(stargazer, n=2, attrName=active, attrValues=[true]) * Results are the top two users (rows) which have the "active" attribute set to "true", sorted by the number of bits set (repositories that they've starred). -#### Range Queries - -**Spec:** - -``` -Range(=, , ) -``` - -**Description:** - -Similar to `Row`, but only returns bits which were set with timestamps -between the given `start` (first) and `end` (second) timestamps. - -**Result Type:** object with attrs and bits - - -**Examples:** - -Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: -```request -Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -This example assumes timestamps have been set on some bits. - -* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. - - -#### Range (BSI) - -**Spec:** - -``` -Range([ ] ) -``` - -**Description:** - -The `Range` query is overloaded to work on `integer` values as well as `timestamp` values. -Returns bits that are true for the comparison operator. - -**Result Type:** object with attrs and columns - -**Examples:** - -In our source data, commitactivity was counted over the last year. -The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): - -```request -Range(commitactivity > 100) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -* columns are repositories which had at least 100 commits in the last year. - -BSI range queries support the following operators: - - Operator | Name | Value -----------|-------------------------------|-------------------- - `>` | greater-than, GT | integer - `<` | less-than, LT | integer - `<=` | less-than-or-equal-to, LTE | integer - `>=` | greater-than-or-equal-to, GTE | integer - `==` | equal-to, EQ | integer - `!=` | not-equal-to, NEQ | integer or `null` - -`<`, and `<=` can be chained together to represent a bounded interval. For example: - -```request -Range(50 < commitactivity < 150) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported. #### Min @@ -762,7 +763,7 @@ Modifies the given query as follows: * `excludeRowAttrs`: Exclude row attributes from the result (Default: `false`). * `shards`: Run the query using only the data from the given shards. By default, the entire data set (i.e. data from all shards) is used. -**Result Type:** Same result type as . +**Result Type:** Same result type as ``. **Examples:** @@ -781,3 +782,111 @@ Options(Row(f1=10), shards=[0, 2]) ```response {"attrs":{},"columns":[100, 2097152]} ``` + +**Spec:** + +``` +Rows(field=, previous=, limit=, column=) +``` + +**Description:** + +Rows returns a list of row IDs in the given field which have at least one bit +set. The field argument is mandatory, the others are optional. + +If `previous` is given, rows prior to and including the specified row ID or +key will not be returned. If `column` is given, only rows which have a set bit +in the given column will be returned. `previous` or `column` must be strings if +and only if the field or index respectively is using key translation. If `limit` +is given, the number of rowIDs returned will be less than or equal to +`limit`. The combination of `limit` and `previous` allows for paging over large +result sets. Results are always ordered, so setting `previous` as the last +result of the previous request will start from the next available row. + + +**Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.` + +**Examples:** + +Without keys: +```request +Rows(field=blah) +``` +```response +{"rows":[1,9,39]} +``` + +With keys: +```request +Rows(field=blahk) +``` +```response +{"rows":null,"keys":["haha","zaaa","traa"]} +``` + + +**Spec:** + +``` +GroupBy(, [RowsCall...], limit=, filter=) +``` + +**Description:** + +GroupBy returns the count of the intersection of every combination of rows +taking one row each from the specified `Rows` calls. It returns only those +combinations for which the count is greater than 0. + +The optional `filter` argument takes any type of `Row` query (e.g. Row, Union, + Intersect, etc.) which will be intersected with each result prior to returning + the count. This is analagous to a WHERE clause applied to a relational GROUP BY + query. + +The optional `limit` argument limits the number of results returned. The results +are ordered, so as long as the data isn't changing, the same query will return +the same result set. + +Paging through results is supported by passing the `previous` argument to each +of the `Rows` calls in the GroupBy. Take the last result from your previous +`GroupBy` query, and pass each row ID in that result as the `previous` argument +to each of the respective `Rows` queries in your next `GroupBy` query. + +**Result Type:** Array of "groups". Each group is an object with a group key and +a count key. The count is an integer, and the group is an array of objects which +specify the field and row for each row that was intersected to get that result. + +**Examples:** + +A single `Rows` query. +```request +GroupBy(Rows(field=blah)) +``` +```response +[{"group":[{"field":"blah","rowID":1}],"count":1}, +{"group":[{"field":"blah","rowID":9}],"count":1}, +{"group":[{"field":"blah","rowID":39}],"count":1}] +``` + +With two `Rows` queries - one with IDs and one with keys. +```request +GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) +``` +```response +[{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1}, + {"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"traa"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"haha"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"traa"}],"count":1}, + {"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"haha"}],"count":1}] +``` + +Getting the rest of the results from the previous example (paging). +```request +GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7) +``` + +```response +[{"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"traa"}],"count":1}] +``` diff --git a/docs/tutorials.md b/docs/tutorials.md index 47344db4f..01e525362 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -444,7 +444,7 @@ Refer to the [Docker documentation](https://docs.docker.com) to see your options #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Row`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request @@ -455,7 +455,7 @@ curl localhost:10101/index/patients \ {"success":true} ``` -In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` field. +In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `patients` index. ``` request curl localhost:10101/index/patients/field/age \ -X POST \ @@ -529,22 +529,22 @@ Assuming we have a file called `ages.csv` that is structured like this: ``` where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command: ``` -pilosa import -i patients -f measurements --field age ages.csv +pilosa import -i patients --field age ages.csv ``` Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. -In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. +In order to find all patients over the age of 40, then simply run a `Row` query against the `age` field. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Range(age > 40)' + -d 'Row(age > 40)' ``` ``` response {"results":[{"attrs":{},"columns":[2,6,9]}]} ``` -You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. +You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: ``` request @@ -561,7 +561,7 @@ You can also provide a filter to the `Sum()` function to find the average age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(Range(age > 40), field="age")' + -d 'Sum(Row(age > 40), field="age")' ``` ``` response {"results":[{"value":191,"count":3}]} @@ -583,7 +583,7 @@ You can also provide a filter to the `Min()` function to find the minimum age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(Range(age > 40), field="age")' + -d 'Min(Row(age > 40), field="age")' ``` ``` response {"results":[{"value":57,"count":1}]} @@ -604,7 +604,7 @@ You can also provide a filter to the `Max()` function to find the maximum age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(Range(age < 40), field="age")' + -d 'Max(Row(age < 40), field="age")' ``` ``` response {"results":[{"value":34,"count":1}]} diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 3c7629a65..d40db1fb9 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -249,6 +249,22 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeBlockDataResponse(msg, mt) return nil + case *pilosa.TranslateKeysRequest: + msg := &internal.TranslateKeysRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateKeysRequest") + } + decodeTranslateKeysRequest(msg, mt) + return nil + case *pilosa.TranslateKeysResponse: + msg := &internal.TranslateKeysResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateKeysResponse") + } + decodeTranslateKeysResponse(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -308,6 +324,10 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeBlockDataRequest(mt) case *pilosa.BlockDataResponse: return encodeBlockDataResponse(mt) + case *pilosa.TranslateKeysRequest: + return encodeTranslateKeysRequest(mt) + case *pilosa.TranslateKeysResponse: + return encodeTranslateKeysResponse(mt) } return nil } @@ -437,7 +457,7 @@ func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstru Node: encodeNode(m.Node), Coordinator: encodeNode(m.Coordinator), Sources: encodeResizeSources(m.Sources), - Schema: encodeSchema(m.Schema), + NodeStatus: encodeNodeStatus(m.NodeStatus), ClusterStatus: encodeClusterStatus(m.ClusterStatus), } } @@ -695,6 +715,20 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac return &internal.RecalculateCaches{} } +func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse { + return &internal.TranslateKeysResponse{ + IDs: response.IDs, + } +} + +func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest { + return &internal.TranslateKeysRequest{ + Index: request.Index, + Field: request.Field, + Keys: request.Keys, + } +} + func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID m.Node = &pilosa.Node{} @@ -703,8 +737,8 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeIns decodeNode(ri.Coordinator, m.Coordinator) m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) decodeResizeSources(ri.Sources, m.Sources) - m.Schema = &pilosa.Schema{} - decodeSchema(ri.Schema, m.Schema) + m.NodeStatus = &pilosa.NodeStatus{} + decodeNodeStatus(ri.NodeStatus, m.NodeStatus) m.ClusterStatus = &pilosa.ClusterStatus{} decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) } @@ -999,6 +1033,16 @@ func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { } } +func decodeTranslateKeysRequest(pb *internal.TranslateKeysRequest, m *pilosa.TranslateKeysRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Keys = pb.Keys +} + +func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.TranslateKeysResponse) { + m.IDs = pb.IDs +} + // QueryResult types. const ( queryResultTypeNil uint32 = iota @@ -1103,8 +1147,13 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { other := make([]pilosa.FieldRow, len(a)) for i := range a { - other[i].Field = a[i].Field - other[i].RowID = a[i].RowID + fr := a[i] + other[i].Field = fr.Field + if fr.RowKey == "" { + other[i].RowID = fr.RowID + } else { + other[i].RowKey = fr.RowKey + } } return other } @@ -1182,9 +1231,17 @@ func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { other := make([]*internal.FieldRow, len(a)) for i := range a { - other[i] = &internal.FieldRow{ - Field: a[i].Field, - RowID: a[i].RowID, + fr := a[i] + if fr.RowKey == "" { + other[i] = &internal.FieldRow{ + Field: fr.Field, + RowID: fr.RowID, + } + } else { + other[i] = &internal.FieldRow{ + Field: fr.Field, + RowKey: fr.RowKey, + } } } return other diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index c2f95c0c7..fe357d933 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -177,6 +177,15 @@ func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterato }, found } +func (btc *bTreeContainers) Repair() { + e, _ := btc.tree.Seek(0) + _, c, err := e.Next() + for err != io.EOF { + c.Repair() + _, c, err = e.Next() + } +} + type btcIterator struct { e *enumerator key uint64 diff --git a/executor.go b/executor.go index edff9f0e6..28290288c 100644 --- a/executor.go +++ b/executor.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "log" "sort" "time" @@ -87,6 +88,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar resp := QueryResponse{} + // Check for query cancellation. + if err := validateQueryContext(ctx); err != nil { + return resp, err + } + // Verify that an index is set. if index == "" { return resp, ErrIndexRequired @@ -112,12 +118,16 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } } results, err := e.execute(ctx, index, q, shards, opt) if err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } resp.Results = results @@ -159,6 +169,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } } @@ -217,6 +229,10 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + v, err := e.executeCall(ctx, index, call, shards, opt) if err != nil { return nil, err @@ -231,7 +247,9 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") defer span.Finish() - if err := e.validateCallArgs(c); err != nil { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } else if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } indexTag := fmt.Sprintf("index:%s", index) @@ -475,11 +493,11 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return nil, errors.Wrap(err, "map reduce") } - // Attach attributes for Row() calls. + // Attach attributes for non-BSI Row() calls. // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. row, _ := other.(*Row) - if c.Name == "Row" { + if c.Name == "Row" && !c.HasConditionArg() { if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { @@ -521,18 +539,20 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // executeBitmapCallShard executes a bitmap call for a single shard. func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") defer span.Finish() switch c.Name { - case "Row": - return e.executeBitmapShard(ctx, index, c, shard) + case "Row", "Range": + return e.executeRowShard(ctx, index, c, shard) case "Difference": return e.executeDifferenceShard(ctx, index, c, shard) case "Intersect": return e.executeIntersectShard(ctx, index, c, shard) - case "Range": - return e.executeRangeShard(ctx, index, c, shard) case "Union": return e.executeUnionShard(ctx, index, c, shard) case "Xor": @@ -886,6 +906,10 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } else if hasLimit { limit = int(lim) } + filter, _, err := c.CallArg("filter") + if err != nil { + return nil, err + } // perform necessary Rows queries (any that have limit or columns args) - // TODO, call async? would only help if multiple Rows queries had a column @@ -894,7 +918,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { if child.Name != "Rows" { - return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", c.Name) + return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name) } _, hasLimit, err := child.UintArg("limit") if err != nil { @@ -917,7 +941,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, shard, childRows) + return e.executeGroupByShard(ctx, index, c, filter, shard, childRows) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { @@ -977,7 +1001,7 @@ func (fr FieldRow) MarshalJSON() ([]byte, error) { } func (fr FieldRow) String() string { - return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) + return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } type GroupCount struct { @@ -1030,8 +1054,15 @@ func (g GroupCount) Compare(o GroupCount) int { return 0 } -func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - iter, err := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) { + var filterRow *Row + if filter != nil { + if filterRow, err = e.executeBitmapCallShard(ctx, index, filter, shard); err != nil { + return nil, errors.Wrapf(err, "executing group by filter for shard %d", shard) + } + } + + iter, err := newGroupByIterator(childRows, c.Children, filterRow, index, shard, e.Holder) if err != nil { return nil, errors.Wrapf(err, "getting group by iterator for shard %d", shard) } @@ -1140,10 +1171,19 @@ func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call return frag.rows(start, filters...), nil } -func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapShard") +func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard") defer span.Finish() + if c.Name == "Range" { + log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.") + } + + // Handle bsiGroup ranges differently. + if c.HasConditionArg() { + return e.executeRowBSIGroupShard(ctx, index, c, shard) + } + // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { @@ -1167,93 +1207,43 @@ func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.fragment(index, fieldName, viewStandard, shard) - if frag == nil { - return NewRow(), nil - } - return frag.row(rowID), nil -} - -// executeIntersectShard executes a intersect() call for a local shard. -func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") - defer span.Finish() - - var other *Row - if len(c.Children) == 0 { - return nil, fmt.Errorf("empty Intersect query is currently not supported") - } - for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) - if err != nil { - return nil, err - } - - if i == 0 { - other = row - } else { - other = other.Intersect(row) + // Parse "from" time, if set. + var fromTime time.Time + if _, ok := c.Args["from"]; ok { + switch v := c.Args["from"].(type) { + case string: + if fromTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'from' time") + } + case int64: + fromTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'from' arg must be a timestamp") } } - other.invalidateCount() - return other, nil -} -// executeRangeShard executes a range() call for a local shard. -func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeRangeShard") - defer span.Finish() - - // Handle bsiGroup ranges differently. - if c.HasConditionArg() { - return e.executeBSIGroupRangeShard(ctx, index, c, shard) + // Parse "to" time, if set. + var toTime time.Time + if _, ok := c.Args["to"]; ok { + switch v := c.Args["to"].(type) { + case string: + if toTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'to' time") + } + case int64: + toTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'to' arg must be a timestamp") + } } - // Parse field. - fieldName, err := c.FieldArg() - if err != nil { - return nil, errors.New("Range() argument required: field") - } - - // Retrieve column label. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - - // Retrieve base field. - f := idx.Field(fieldName) - if f == nil { - return nil, ErrFieldNotFound - } - - // Read row & column id. - rowID, rowOK, err := c.UintArg(fieldName) - if err != nil { - return nil, fmt.Errorf("executeRangeShard - reading row: %v", err) - } - if !rowOK { - return nil, fmt.Errorf("Range() must specify %q", rowLabel) - } - - // Parse start time. - startTimeStr, ok := c.Args["_start"].(string) - if !ok { - return nil, errors.New("Range() start time required") - } - startTime, err := time.Parse(TimeFormat, startTimeStr) - if err != nil { - return nil, errors.New("cannot parse Range() start time") - } - - // Parse end time. - endTimeStr, ok := c.Args["_end"].(string) - if !ok { - return nil, errors.New("Range() end time required") - } - endTime, err := time.Parse(TimeFormat, endTimeStr) - if err != nil { - return nil, errors.New("cannot parse Range() end time") + // Simply return row if times are not set. + if c.Name == "Row" && fromTime.IsZero() && toTime.IsZero() { + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { + return NewRow(), nil + } + return frag.row(rowID), nil } // If no quantum exists then return an empty bitmap. @@ -1262,9 +1252,16 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C return &Row{}, nil } + // Set maximum "to" value if only "from" is set. We don't need to worry + // about setting the minimum "from" since it is the zero value if omitted. + if toTime.IsZero() { + // Set the end timestamp to current time + 1 day, in order to account for timezone differences. + toTime = time.Now().AddDate(0, 0, 1) + } + // Union bitmaps across all time-based views. row := &Row{} - for _, view := range viewsByTimeRange(viewStandard, startTime, endTime, q) { + for _, view := range viewsByTimeRange(viewStandard, fromTime, toTime, q) { f := e.Holder.fragment(index, fieldName, view, shard) if f == nil { continue @@ -1273,18 +1270,19 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C } f.Stats.Count("range", 1, 1.0) return row, nil + } -// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBSIGroupRangeShard") +// executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. +func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard") defer span.Finish() // Only one conditional should be present. if len(c.Args) == 0 { - return nil, errors.New("Range(): condition required") + return nil, errors.New("Row(): condition required") } else if len(c.Args) > 1 { - return nil, errors.New("Range(): too many arguments") + return nil, errors.New("Row(): too many arguments") } // Extract conditional. @@ -1293,7 +1291,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { - return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) + return nil, fmt.Errorf("Row(): %q: expected condition argument, got %v", k, v) } fieldName, cond = k, vv } @@ -1305,7 +1303,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // EQ null (not implemented: flip frag.NotNull with max ColumnID) // NEQ null frag.NotNull() - // BETWEEN a,b(in) BETWEEN/frag.RangeBetween() + // BETWEEN a,b(in) BETWEEN/frag.RowBetween() // BETWEEN a,b(out) BETWEEN/frag.NotNull() // EQ frag.RangeOp // NEQ frag.RangeOp @@ -1335,11 +1333,11 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // Only support two integers for the between operation. if len(predicates) != 2 { - return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values") + return nil, errors.New("Row(): BETWEEN condition requires exactly two integer values") } // The reason we don't just call: - // return f.RangeBetween(fieldName, predicates[0], predicates[1]) + // return f.RowBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be shard-specific. // Find bsiGroup. @@ -1372,7 +1370,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // Only support integers for now. value, ok := cond.Value.(int64) if !ok { - return nil, errors.New("Range(): conditions only support integer values") + return nil, errors.New("Row(): conditions only support integer values") } // Find bsiGroup. @@ -1408,6 +1406,31 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, } } +// executeIntersectShard executes a intersect() call for a local shard. +func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") + defer span.Finish() + + var other *Row + if len(c.Children) == 0 { + return nil, fmt.Errorf("empty Intersect query is currently not supported") + } + for i, input := range c.Children { + row, err := e.executeBitmapCallShard(ctx, index, input, shard) + if err != nil { + return nil, err + } + + if i == 0 { + other = row + } else { + other = other.Intersect(row) + } + } + other.invalidateCount() + return other, nil +} + // executeUnionShard executes a union() call for a local shard. func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") @@ -1984,7 +2007,13 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) - for _, c := range calls { + for i, c := range calls { + if i%10 == 0 { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + } + field, ok := c.Args["_field"].(string) if !ok { return nil, errors.New("SetRowAttrs() field required") @@ -2145,7 +2174,7 @@ func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (m loop: for _, shard := range shards { - for _, node := range e.Cluster.shardNodes(index, shard) { + for _, node := range e.Cluster.ShardNodes(index, shard) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], shard) continue loop @@ -2567,6 +2596,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return result, nil } +// validateQueryContext returns a query-appropriate error if the context is done. +func validateQueryContext(ctx context.Context) error { + select { + case <-ctx.Done(): + switch err := ctx.Err(); err { + case context.Canceled: + return ErrQueryCancelled + case context.DeadlineExceeded: + return ErrQueryTimeout + default: + return err + } + default: + return nil + } +} + // errShardUnavailable is a marker error if no nodes are available. var errShardUnavailable = errors.New("shard unavailable") @@ -2705,22 +2751,32 @@ type groupByIterator struct { // fields and then sets the row ids. fields []FieldRow done bool + + // Optional filter row to intersect against first level of values. + filter *Row } // newGroupByIterator initializes a new groupByIterator. -func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) (*groupByIterator, error) { +func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, index string, shard uint64, holder *Holder) (*groupByIterator, error) { gbi := &groupByIterator{ rowIters: make([]*rowIterator, len(children)), rows: make([]struct { row *Row id uint64 }, len(children)), + filter: filter, fields: make([]FieldRow, len(children)), } ignorePrev := false for i, call := range children { - fieldName := call.Args["field"].(string) // this has already been validated by this point + fieldName, ok := call.Args["field"].(string) + if !ok { + return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"]) + } + if holder.Field(index, fieldName) == nil { + return nil, ErrFieldNotFound + } gbi.fields[i].Field = fieldName // Fetch fragment. frag := holder.fragment(index, fieldName, viewStandard, shard) @@ -2774,6 +2830,11 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha } } + // Apply filter to first level, if available. + if gbi.filter != nil && len(gbi.rows) > 0 { + gbi.rows[0].row = gbi.rows[0].row.Intersect(gbi.filter) + } + for i := 1; i < len(gbi.rows)-1; i++ { gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row) } @@ -2784,32 +2845,49 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha // nextAtIdx is a recursive helper method for getting the next row for the field // at index i, and then updating the rows in the "higher" fields if it wraps. func (gbi *groupByIterator) nextAtIdx(i int) { - nr, rowID, wrapped := gbi.rowIters[i].Next() - if nr == nil { - gbi.done = true - return + // loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed. + for { + nr, rowID, wrapped := gbi.rowIters[i].Next() + if nr == nil { + gbi.done = true + return + } + if wrapped && i != 0 { + gbi.nextAtIdx(i - 1) + } + if i == 0 && gbi.filter != nil { + gbi.rows[i].row = nr.Intersect(gbi.filter) + } else if i == 0 || i == len(gbi.rows)-1 { + gbi.rows[i].row = nr + } else { + gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) + } + gbi.rows[i].id = rowID + + if !gbi.rows[i].row.IsEmpty() { + break + } } - if wrapped && i != 0 { - gbi.nextAtIdx(i - 1) - } - if i != 0 && i != len(gbi.rows)-1 { - gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) - } else { - gbi.rows[i].row = nr - } - gbi.rows[i].id = rowID } // Next returns a GroupCount representing the next group by record. When there // are no more records it will return an empty GroupCount and done==true. func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { - if gbi.done { - return ret, true - } - if len(gbi.rows) == 1 { - ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() - } else { - ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) + // loop until we find a result with count > 0 + for { + if gbi.done { + return ret, true + } + if len(gbi.rows) == 1 { + ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() + } else { + ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) + } + if ret.Count == 0 { + gbi.nextAtIdx(len(gbi.rows) - 1) + continue + } + break } ret.Group = make([]FieldRow, len(gbi.rows)) @@ -2819,6 +2897,7 @@ func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { } // set up for next call + gbi.nextAtIdx(len(gbi.rows) - 1) return ret, false diff --git a/executor_test.go b/executor_test.go index 5b7623006..2d155bd97 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1478,23 +1478,29 @@ func TestExecutor_Execute_Sum(t *testing.T) { } // Ensure a range query can be executed. -func TestExecutor_Execute_Range(t *testing.T) { +func TestExecutor_Execute_Row_Range(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { - writeQuery := ` + // Create a timestamp just out of the current date + 1 day timestamp (default end timestamp). + nextDayExclusive := time.Now().AddDate(0, 0, 2) + + writeQuery := fmt.Sprintf(` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) Set(5, f=1, 2000-02-01T00:00) Set(6, f=1, 2001-01-01T00:00) Set(7, f=1, 2002-01-01T02:00) + Set(8, f=1, %s) Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00)` + Set(2, f=10, 2001-01-01T00:00)`, nextDayExclusive.Format("2006-01-02T15:04")) readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00)`, + `Row(f=1, to=2002-01-01T02:00)`, `Clear( 2, f=1)`, - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) @@ -1505,8 +1511,20 @@ func TestExecutor_Execute_Range(t *testing.T) { } }) + t.Run("From", func(t *testing.T) { + if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("To", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Clear", func(t *testing.T) { - if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + if columns := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) @@ -1525,9 +1543,9 @@ func TestExecutor_Execute_Range(t *testing.T) { Set("two", f=1, 2002-02-01T00:00) Set("two", f=10, 2001-01-01T00:00)` readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear("two", f=1)`, - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, @@ -1559,9 +1577,9 @@ func TestExecutor_Execute_Range(t *testing.T) { Set(2, f="foo", 2002-02-01T00:00) Set(2, f="bar", 2001-01-01T00:00)` readQueries := []string{ - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear( 2, f="foo")`, - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, nil, @@ -1594,9 +1612,182 @@ func TestExecutor_Execute_Range(t *testing.T) { Set("two", f="foo", 2002-02-01T00:00) Set("two", f="bar", 2001-01-01T00:00)` readQueries := []string{ - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear("two", f="foo")`, - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + }) + + t.Run("UnixTimestamp", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Row(f=1, from=946598400, to=1009854000)`, + `Clear( 2, f=1)`, + `Row(f=1, from=946598400, to=1009854000)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) +} + +// Ensure a range query can be executed. +func TestExecutor_Execute_Range_Deprecated(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear( 2, f=1)`, + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f=1, 1999-12-31T00:00) + Set("three", f=1, 2000-01-01T00:00) + Set("four", f=1, 2000-01-02T00:00) + Set("five", f=1, 2000-02-01T00:00) + Set("six", f=1, 2001-01-01T00:00) + Set("seven", f=1, 2002-01-01T02:00) + + Set("two", f=1, 1999-12-30T00:00) + Set("two", f=1, 2002-02-01T00:00) + Set("two", f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear("two", f=1)`, + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) + + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear( 2, f="foo")`, + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f="foo", 1999-12-31T00:00) + Set("three", f="foo", 2000-01-01T00:00) + Set("four", f="foo", 2000-01-02T00:00) + Set("five", f="foo", 2000-02-01T00:00) + Set("six", f="foo", 2001-01-01T00:00) + Set("seven", f="foo", 2002-01-01T02:00) + + Set("two", f="foo", 1999-12-30T00:00) + Set("two", f="foo", 2002-02-01T00:00) + Set("two", f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear("two", f="foo")`, + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, @@ -1617,8 +1808,202 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } -// Ensure a Range(bsiGroup) query can be executed. -func TestExecutor_Execute_BSIGroupRange(t *testing.T) { +// Ensure a Row(bsiGroup) query can be executed. +func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) + + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) + `}); err != nil { + t.Fatal(err) + } + + t.Run("EQ", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("NEQ", func(t *testing.T) { + // NEQ null + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ - + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + //t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) + } + }) + + t.Run("LT", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo < 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTE", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GT", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GTE", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BETWEEN", func(t *testing.T) { + tests := []struct { + q string + exp bool + }{ + {q: `Row(0 < other < 1000)`, exp: false}, + {q: `Row(0 <= other < 1000)`, exp: false}, + {q: `Row(0 <= other <= 1000)`, exp: true}, + {q: `Row(0 < other <= 1000)`, exp: true}, + + {q: `Row(1000 < other < 1000)`, exp: false}, + {q: `Row(1000 <= other < 1000)`, exp: false}, + {q: `Row(1000 <= other <= 1000)`, exp: true}, + {q: `Row(1000 < other <= 1000)`, exp: false}, + + {q: `Row(1000 < other < 2000)`, exp: false}, + {q: `Row(1000 <= other < 2000)`, exp: true}, + {q: `Row(1000 <= other <= 2000)`, exp: true}, + {q: `Row(1000 < other <= 2000)`, exp: false}, + } + for i, test := range tests { + t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { + var expected = []uint64{} + if test.exp { + expected = []uint64{0} + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result for query: %s", test.q) + } + }) + } + + }) + + // Ensure that the NotNull code path gets run. + t.Run("NotNull", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BelowMin", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 0)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("AboveMax", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTAboveMax", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge < 200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) + } + }) + + t.Run("GTBelowMin", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) + } + }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { + t.Fatal(err) + } + }) +} + +// Ensure a Range(bsiGroup) query can be executed. (Deprecated) +func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -1730,14 +2115,14 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1907,7 +2292,7 @@ Set(4500001, fn=4) {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, } results := res.Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) } }) } @@ -2010,7 +2395,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { Set(2, f=10, 2001-01-01T00:00) ` clearColumn := `Clear( 2, f=1)` - rangeCheckQuery := `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)` + rangeCheckQuery := `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)` for i, tt := range rangeTests { t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) { @@ -2163,10 +2548,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after Not: %+v", bits) } // Reopen cluster to ensure existence field is reloaded. @@ -2174,10 +2559,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns after reopen: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after reopen: %+v", bits) } }) } @@ -2346,10 +2731,10 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { Set(2, f=1, 2002-02-01T00:00) Set(2, f=10, 2001-01-01T00:00)` readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`, `ClearRow(f=1)`, - `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, - `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`, + `Row(f=10, from=1999-12-31T00:00, to=2003-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{TrackExistence: true}, @@ -2674,6 +3059,66 @@ func TestExecutor_Execute_Rows(t *testing.T) { } } +func TestExecutor_Execute_Query_Error(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + + tests := []struct { + query string + error string + }{ + { + query: "GroupBy(Rows())", + error: "Rows call must have 'field' argument", + }, + { + query: "GroupBy(Rows(field=true))", + error: "Rows call must have 'field' argument", + }, + { + query: "GroupBy(Rows(field=\"true\"))", + error: "field not found", + }, + { + query: "GroupBy(Rows(field=1))", + error: "Rows call must have 'field' argument", + }, + { + query: "GroupBy(Rows(field))", + error: "parse error", + }, + { + query: "GroupBy(Rows(field=general, limit=-1))", + error: "must be positive, but got", + }, + { + query: "GroupBy(Rows(field=general), limit=-1)", + error: "must be positive, but got", + }, + { + query: "GroupBy(Rows(field=general), filter=Rows(field=general))", + error: "unknown call: Rows", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: test.query, + }) + if err == nil { + t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r) + } + if !strings.Contains(err.Error(), test.error) { + t.Fatalf("unexpected error message: %s", err.Error()) + } + }) + } + +} + func TestExecutor_Execute_Rows_Keys(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -2842,7 +3287,17 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) + }) + + t.Run("Filter", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) }) t.Run("check field offset no limit", func(t *testing.T) { @@ -2852,7 +3307,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("check field offset limit", func(t *testing.T) { @@ -2861,7 +3316,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) @@ -2882,7 +3337,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) // set the same bits in a single shard in three fields @@ -2915,7 +3370,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("test previous is last result", func(t *testing.T) { @@ -2930,7 +3385,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) // test multiple shards with distinct results (different rows) and same @@ -2958,7 +3413,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { @@ -2969,7 +3424,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { @@ -2980,7 +3435,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 3}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) c.CreateField(t, "i", pilosa.IndexOptions{}, "na") @@ -3005,7 +3460,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 1}}, Count: 2}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) @@ -3050,8 +3505,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } expected[63].Count = 5 - checkGroupBy(t, expected, totalResults) + test.CheckGroupBy(t, expected, totalResults) }) + + // test row keys + c.CreateField(t, "i", pilosa.IndexOptions{}, "generalk", pilosa.OptFieldKeys()) + c.CreateField(t, "i", pilosa.IndexOptions{}, "subk", pilosa.OptFieldKeys()) + c.Query(t, "i", ` + Set(0, generalk="ten") + Set(1, generalk="ten") + Set(1001, generalk="ten") + Set(2, generalk="eleven") + Set(1002, generalk="eleven") + Set(2, generalk="twelve") + Set(1002, generalk="twelve") + + Set(0, subk="one-hundred") + Set(1, subk="one-hundred") + Set(3, subk="one-hundred") + Set(1001, subk="one-hundred") + Set(2, subk="one-hundred-ten") + Set(0, subk="one-hundred-ten") + `) + + t.Run("test row keys", func(t *testing.T) { + // the execututor returns row IDs when the field has keys, so they should be included in the target. + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 1, RowKey: "one-hundred"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 2, RowKey: "eleven"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + + }) + } for size := range []int{1, 3} { t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { @@ -3114,18 +3604,9 @@ func BenchmarkGroupBy(b *testing.B) { } -func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { - if len(results) != len(expected) { - t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) - } - for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { - t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) - } - } -} - func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse { + t.Helper() + if indexOptions == nil { indexOptions = &pilosa.IndexOptions{} } diff --git a/field.go b/field.go index b71f9ca7e..669700574 100644 --- a/field.go +++ b/field.go @@ -1124,7 +1124,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { - return errors.Wrap(err, "creating view") + return errors.Wrap(err, "creating fragment") } if err := frag.bulkImport(data.RowIDs, data.ColumnIDs, options); err != nil { diff --git a/fragment.go b/fragment.go index 028d105b4..4f71809c9 100644 --- a/fragment.go +++ b/fragment.go @@ -28,6 +28,7 @@ import ( "math" "os" "sort" + "strings" "sync" "syscall" "time" @@ -2315,46 +2316,38 @@ func (s *fragmentSyncer) syncBlock(id int) error { // Write updates to remote blocks. for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] - count := 0 - // Ignore if there are no differences. - if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { - continue - } - - // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.columnIDs) + len(clear.columnIDs) - maxWrites := s.Cluster.maxWritesPerRequest - if maxWrites <= 0 { - maxWrites = 5000 - } - buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) - - // Only sync the standard block. - for j := 0; j < len(set.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) - count++ - } - for j := 0; j < len(clear.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) - count++ - } - - // Iterate over the buffers. - for k := 0; k < len(buffers); k++ { - // Verify sync is not prematurely closing. - if s.isClosing() { - return nil - } - - // Execute query. - queryRequest := &QueryRequest{ - Query: buffers[k].String(), - Remote: true, - } - _, err := s.Cluster.InternalClient.QueryNode(ctx, uris[i], f.index, queryRequest) + // Handle Sets. + if len(set.columnIDs) > 0 { + setData, err := bitsToRoaringData(set) if err != nil { - return errors.Wrap(err, "executing") + return errors.Wrap(err, "converting bits to roaring data (set)") + } + + setReq := &ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{cleanViewName(f.view): setData}, + } + + if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, setReq); err != nil { + return errors.Wrap(err, "sending roaring data (set)") + } + } + + // Handle Clears. + if len(clear.columnIDs) > 0 { + clearData, err := bitsToRoaringData(clear) + if err != nil { + return errors.Wrap(err, "converting bits to roaring data (clear)") + } + + clearReq := &ImportRoaringRequest{ + Clear: true, + Views: map[string][]byte{"": clearData}, + } + + if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, clearReq); err != nil { + return errors.Wrap(err, "sending roaring data (clear)") } } } @@ -2362,6 +2355,39 @@ func (s *fragmentSyncer) syncBlock(id int) error { return nil } +// cleanViewName converts a viewname into the equivalent +// string required by the external api. Because views are +// not exposed externally, the conversion looks like this: +// "standard" -> "" +// "standard_YYYYMMDD" -> "YYYYMMDD" +// "other" -> "other" (there is currently not a use for this) +func cleanViewName(v string) string { + viewPrefix := viewStandard + "_" + if strings.HasPrefix(v, viewPrefix) { + return v[len(viewPrefix):] + } else if v == viewStandard { + return "" + } + return v +} + +// bitsToRoaringData converts a pairSet into a roaring.Bitmap +// which represents the data within a single shard. +func bitsToRoaringData(ps pairSet) ([]byte, error) { + bmp := roaring.NewBitmap() + for j := 0; j < len(ps.columnIDs); j++ { + bmp.DirectAdd(ps.rowIDs[j]*ShardWidth + (ps.columnIDs[j] % ShardWidth)) + } + + var buf bytes.Buffer + _, err := bmp.WriteTo(&buf) + if err != nil { + return nil, errors.Wrap(err, "writing to buffer") + } + + return buf.Bytes(), nil +} + func madvise(b []byte, advice int) error { // nolint: unparam _, _, err := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice)) if err != 0 { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index b36bf3380..cca88dda1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -20,6 +20,8 @@ import ( "fmt" "io/ioutil" "math" + "math/rand" + "os" "reflect" "sort" "testing" @@ -37,12 +39,17 @@ var ( // In order to generate the sample fragment file, // run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") + TempDir = "" ) +func init() { // nolint: gochecknoinits + flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") +} + // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(120, 1); err != nil { @@ -73,7 +80,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -100,7 +107,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -127,7 +134,7 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 7, "") - defer f.Close() + defer f.Clean(t) rowID := uint64(1000) @@ -176,7 +183,7 @@ func TestFragment_SetRow(t *testing.T) { func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -204,7 +211,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Overwrite", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -232,7 +239,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Clear", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -260,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("NotExists", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 10, 20); err != nil { @@ -290,7 +297,7 @@ func TestFragment_SetValue(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. m := make(map[uint64]int64) @@ -328,7 +335,7 @@ func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -367,7 +374,7 @@ func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -441,7 +448,7 @@ func TestFragment_Range(t *testing.T) { t.Run("EQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -464,7 +471,7 @@ func TestFragment_Range(t *testing.T) { t.Run("NEQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -487,7 +494,7 @@ func TestFragment_Range(t *testing.T) { t.Run("LT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -535,7 +542,7 @@ func TestFragment_Range(t *testing.T) { t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -583,7 +590,7 @@ func TestFragment_Range(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -633,7 +640,7 @@ func TestFragment_Range(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -662,7 +669,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(100, 20); err != nil { @@ -691,7 +698,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) f.mustSetBits(101, 1) @@ -713,7 +720,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) @@ -743,7 +750,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -774,7 +781,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) // Create an intersecting input row. src := NewRow( @@ -812,7 +819,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -833,7 +840,7 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) - defer f.Close() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -881,7 +888,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { if err := f.Open(); err != nil { panic(err) } - defer f.Close() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -914,7 +921,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Retrieve checksum and set bits. orig := f.Checksum() @@ -933,7 +940,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Retrieve initial checksum. var prev []FragmentBlock @@ -971,7 +978,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set bits on a different block. if _, err := f.setBit(100, 1); err != nil { @@ -989,7 +996,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) - defer f.Close() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1074,7 +1081,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f0.Close() + defer f0.Clean(t) // Set and then clear bits on the fragment. if _, err := f0.setBit(1000, 1); err != nil { @@ -1135,7 +1142,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Close() + defer f.Clean(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1148,7 +1155,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(b) f.MaxOpN = math.MaxInt32 // Generate some intersecting data. @@ -1179,7 +1186,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { func TestFragment_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1202,7 +1209,7 @@ func TestFragment_Tanimoto(t *testing.T) { func TestFragment_Zero_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1227,7 +1234,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { func TestFragment_Snapshot_Run(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(1); i < 3; i++ { @@ -1254,7 +1261,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) var cols []uint64 @@ -1368,7 +1375,7 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1404,7 +1411,7 @@ func TestFragment_ImportSet(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) @@ -1501,7 +1508,7 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1620,7 +1627,7 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1664,7 +1671,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Close() + defer f.Clean(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -1680,7 +1687,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(b) // Generate some intersecting data. maxX := 1048576 / 2 sz := maxX @@ -1718,7 +1725,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(b) maxX := 1048576 * 5 * 2 sz := maxX rows := make([]uint64, sz) @@ -1746,11 +1753,286 @@ func BenchmarkFragment_Import(b *testing.B) { } } +var ( + rowCases = []uint64{2, 50, 1000, 100000} + colCases = []uint64{20, 1000, 50000, 500000} + concurrencyCases = []int{2, 4, 8, 16} +) + +func BenchmarkImportRoaring(b *testing.B) { + for _, numRows := range rowCases { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) + for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much + b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + b.StartTimer() + err := f.importRoaring(data, false) + if err != nil { + f.Clean(b) + b.Fatalf("import error: %v", err) + } + b.StopTimer() + f.Clean(b) + } + }) + } + } +} + +func BenchmarkImportRoaringConcurrent(b *testing.B) { + for _, numRows := range rowCases { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) + for _, concurrency := range concurrencyCases { + b.Run(fmt.Sprintf("%dRows%dConcurrency", numRows, concurrency), func(b *testing.B) { + b.StopTimer() + frags := make([]*fragment, concurrency) + for i := 0; i < b.N; i++ { + for j := 0; j < concurrency; j++ { + frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) + } + eg := errgroup.Group{} + b.StartTimer() + for j := 0; j < concurrency; j++ { + j := j + eg.Go(func() error { + return frags[j].importRoaring(data, false) + }) + } + err := eg.Wait() + if err != nil { + b.Errorf("importing fragment: %v", err) + } + b.StopTimer() + for j := 0; j < concurrency; j++ { + frags[j].Clean(b) + } + } + }) + } + } +} +func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { + for _, numRows := range rowCases { + for _, numCols := range colCases { + data := getZipfRowsSliceRoaring(numRows, 1) + updata := getUpdataRoaring(numRows, numCols, 1) + for _, concurrency := range concurrencyCases { + b.Run(fmt.Sprintf("%dRows%dCols%dConcurrency", numRows, numCols, concurrency), func(b *testing.B) { + b.StopTimer() + frags := make([]*fragment, concurrency) + for i := 0; i < b.N; i++ { + for j := 0; j < concurrency; j++ { + frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) + frags[j].importRoaring(data, false) + } + eg := errgroup.Group{} + b.StartTimer() + for j := 0; j < concurrency; j++ { + j := j + eg.Go(func() error { + return frags[j].importRoaring(updata, false) + }) + } + err := eg.Wait() + if err != nil { + b.Errorf("importing fragment: %v", err) + } + b.StopTimer() + for j := 0; j < concurrency; j++ { + frags[j].Clean(b) + } + } + }) + } + } + } +} + +func BenchmarkImportStandard(b *testing.B) { + for _, cacheType := range []string{CacheTypeRanked} { + for _, numRows := range rowCases { + rowIDs, columnIDs := getZipfRowsSliceStandard(numRows, 1) + b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + b.StartTimer() + err := f.bulkImport(rowIDs, columnIDs, &ImportOptions{}) + if err != nil { + b.Errorf("import error: %v", err) + } + b.StopTimer() + f.Clean(b) + } + }) + } + } +} + +func BenchmarkImportRoaringUpdate(b *testing.B) { + fileSize := make(map[string]int64) + names := []string{} + for _, cacheType := range []string{CacheTypeRanked} { + for _, numRows := range rowCases { + for _, numCols := range colCases { + data := getZipfRowsSliceRoaring(numRows, 1) + updata := getUpdataRoaring(numRows, numCols, 1) + name := fmt.Sprintf("%s%dRows%dCols", cacheType, numRows, numCols) + names = append(names, name) + b.Run(name, func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + err := f.importRoaring(data, false) + if err != nil { + b.Errorf("import error: %v", err) + } + b.StartTimer() + err = f.importRoaring(updata, false) + if err != nil { + f.Clean(b) + b.Errorf("import error: %v", err) + } + b.StopTimer() + stat, _ := f.file.Stat() + fileSize[name] = stat.Size() + f.Clean(b) + } + }) + + } + } + } + for _, name := range names { + b.Logf("%s: %.2fMB\n", name, float64(fileSize[name])/1024/1024) + } +} + +func TestGetZipfRowsSliceRoaring(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) + data := getZipfRowsSliceRoaring(10, 1) + f.importRoaring(data, false) + if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { + t.Fatalf("unexpected rows: %v", f.rows(0)) + } + for i := uint64(1); i < 10; i++ { + if f.row(i).Count() >= f.row(i-1).Count() { + t.Fatalf("suspect distribution from getZipfRowsSliceRoaring") + } + } +} + +// getZipfRowsSliceRoaring generates a random fragment with the given number of +// rows, and 1 bit set in each column. The row each bit is set in is chosen via +// the Zipf generator, and so will be skewed toward lower row numbers. If this +// is edited to change the data distribution, getZipfRowsSliceStandard should be +// edited as well. +func getZipfRowsSliceRoaring(numRows uint64, seed int64) []byte { + b := roaring.NewBitmap() + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + for i := uint64(0); i < ShardWidth; i++ { + row := z.Uint64() + b.DirectAdd(row*ShardWidth + i) + } + buf := bytes.NewBuffer(make([]byte, 0, 100000)) + _, err := b.WriteTo(buf) + if err != nil { + panic(err) + } + return buf.Bytes() +} + +// getUpdataRoaring gets a byte slice containing a roaring bitmap which +// represents numCols set bits distributed randomly throughout a shard's column +// space and zipfianly throughout numRows rows. +func getUpdataRoaring(numRows, numCols uint64, seed int64) []byte { + b := roaring.NewBitmap() + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + + for i := uint64(0); i < numCols; i++ { + col := uint64(r.Int63n(ShardWidth)) // assuming the number of repeats will be negligible + row := z.Uint64() + b.DirectAdd(row*ShardWidth + col) + } + buf := bytes.NewBuffer(make([]byte, 0, 100000)) + _, err := b.WriteTo(buf) + if err != nil { + panic(err) + } + return buf.Bytes() +} + +// getZipfRowsSliceStandard is the same as getZipfRowsSliceRoaring, but returns +// row and column ids instead of a byte slice containing roaring bitmap data. +func getZipfRowsSliceStandard(numRows uint64, seed int64) (rowIDs, columnIDs []uint64) { + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + rowIDs, columnIDs = make([]uint64, ShardWidth), make([]uint64, ShardWidth) + for i := uint64(0); i < ShardWidth; i++ { + rowIDs[i] = z.Uint64() + columnIDs[i] = i + } + return rowIDs, columnIDs +} + +func BenchmarkFileWrite(b *testing.B) { + for _, numRows := range rowCases { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f, err := ioutil.TempFile(TempDir, "") + if err != nil { + b.Fatalf("getting temp file: %v", err) + } + b.StartTimer() + _, err = f.Write(data) + if err != nil { + b.Fatal(err) + } + err = f.Sync() + if err != nil { + b.Fatal(err) + } + err = f.Close() + if err != nil { + b.Fatal(err) + } + b.StopTimer() + os.Remove(f.Name()) + } + }) + } + +} + ///////////////////////////////////////////////////////////////////// +func (f *fragment) Clean(t testing.TB) { + errc := f.Close() + errf := os.Remove(f.path) + errp := os.Remove(f.cachePath()) + if errc != nil || errf != nil { + t.Fatal("cleaning up fragment: ", errc, errf, errp) + } + // not all fragments have cache files + if errp != nil && !os.IsNotExist(errp) { + t.Fatalf("cleaning up fragment cache: %v", errp) + } +} + // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") + file, err := ioutil.TempFile(TempDir, "pilosa-fragment-") if err != nil { panic(err) } @@ -1811,7 +2093,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) @@ -1838,7 +2120,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) expected := []uint64{1, 2} if _, err := f.setBit(1, 66000); err != nil { @@ -1862,7 +2144,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 100 { @@ -1909,7 +2191,7 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean(t) for num, input := range test { buf := &bytes.Buffer{} bm := roaring.NewBitmap(input...) @@ -1952,7 +2234,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean(t) options := &ImportOptions{} err := f.bulkImport(test.rowIDs, test.colIDs, options) @@ -2086,6 +2368,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2114,11 +2397,11 @@ func TestFragmentRowIterator(t *testing.T) { if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } - f.Close() }) t.Run("skipped rows", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2147,11 +2430,11 @@ func TestFragmentRowIterator(t *testing.T) { if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } - f.Close() }) t.Run("basic wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2172,11 +2455,11 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - f.Close() }) t.Run("skipped rows wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2197,7 +2480,5 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - f.Close() }) - } diff --git a/gossip/gossip.go b/gossip/gossip.go index 8f19203f5..491e14cf2 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -333,15 +333,30 @@ func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { } func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2} } func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2} } func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2} } func (g *eventReceiver) listen() { diff --git a/handler.go b/handler.go index 8fe51dc4c..988435a07 100644 --- a/handler.go +++ b/handler.go @@ -117,3 +117,13 @@ type BlockDataResponse struct { RowIDs []uint64 ColumnIDs []uint64 } + +type TranslateKeysRequest struct { + Index string + Field string + Keys []string +} + +type TranslateKeysResponse struct { + IDs []uint64 +} diff --git a/holder.go b/holder.go index 9e58e8057..471d4bd3e 100644 --- a/holder.go +++ b/holder.go @@ -43,7 +43,7 @@ const ( fileLimit = 262144 // (512^2) // existenceFieldName is the name of the internal field used to store existence values. - existenceFieldName = "exists" + existenceFieldName = "_exists" ) // Holder represents a container for indexes. @@ -289,6 +289,9 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name(), Options: index.Options()} for _, field := range index.Fields() { + if strings.HasPrefix(field.name, "_") { + continue + } fi := &FieldInfo{Name: field.Name(), Options: field.Options()} di.Fields = append(di.Fields, fi) } diff --git a/holder_test.go b/holder_test.go index 3920e4cc7..1bfe4615a 100644 --- a/holder_test.go +++ b/holder_test.go @@ -391,27 +391,26 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { hldr0 := &test.Holder{Holder: c[0].Server.Holder()} hldr1 := &test.Holder{Holder: c[1].Server.Holder()} - // Set data on the local holder. + // Set data on the local holder for node0. t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC) t2 := time.Date(2018, 8, 2, 12, 30, 0, 0, time.UTC) hldr0.SetBitTime("i", "f", 0, 1, &t1) hldr0.SetBitTime("i", "f", 0, 2, &t2) + // Set data on node1. + hldr1.SetBitTime("i", "f", 0, 22, &t2) + err = c[0].Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } - err = c[1].Server.SyncData() - if err != nil { - t.Fatalf("syncing node 1: %v", err) - } // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { if a := hldr.RowTime("i", "f", 0, t1, quantum).Columns(); !reflect.DeepEqual(a, []uint64{1}) { t.Errorf("unexpected columns(%d/0): %+v", i, a) } - if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2, 22}) { t.Errorf("unexpected columns(%d/0): %+v", i, a) } } diff --git a/http/client.go b/http/client.go index b18757a46..78382b81b 100644 --- a/http/client.go +++ b/http/client.go @@ -164,7 +164,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo } return err } - return nil + return errors.Wrap(resp.Body.Close(), "closing response body") } // FragmentNodes returns a list of nodes that own a shard. @@ -705,24 +705,22 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i return nil } -func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { +// RetrieveShardFromURI returns a ReadCloser which contains the data of the +// specified shard from the specified node. Caller *must* close the returned +// ReadCloser or risk leaking goroutines/tcp connections. +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI") defer span.Finish() node := &pilosa.Node{ URI: uri, } - return c.backupShardNode(ctx, index, field, shard, node) -} -func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.backupShardNode") - defer span.Finish() - - u := nodePathToURL(node, "/fragment/data") + u := nodePathToURL(node, "/internal/fragment/data") u.RawQuery = url.Values{ "index": {index}, "field": {field}, + "view": {view}, "shard": {strconv.FormatUint(shard, 10)}, }.Encode() @@ -805,7 +803,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel return err } - return nil + return errors.Wrap(resp.Body.Close(), "closing response body") } // FragmentBlocks returns a list of block checksums for a fragment on a host. @@ -999,15 +997,24 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ req.Header.Set("Accept", "application/json") // Execute request. - _, err = c.executeRequest(req.WithContext(ctx)) - return err + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + return errors.Wrap(resp.Body.Close(), "closing response body") } -// executeRequest executes the given request and checks the Response +// executeRequest executes the given request and checks the Response. For +// responses with non-2XX status, the body is read and closed, and an error is +// returned. If the error is nil, the caller must ensure that the response body +// is closed. func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) { tracing.GlobalTracer.InjectHTTPHeaders(req) resp, err := c.httpClient.Do(req) if err != nil { + if resp != nil { + resp.Body.Close() + } return nil, errors.Wrap(err, "executing request") } if resp.StatusCode < 200 || resp.StatusCode >= 300 { diff --git a/http/client_test.go b/http/client_test.go index fee3754df..2b65f82d4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -710,9 +710,9 @@ func TestClient_ImportKeys(t *testing.T) { t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) } - // Verify Range. + // Verify range. queryRequest := &pilosa.QueryRequest{ - Query: fmt.Sprintf(`Range(%s>10)`, fldName), + Query: fmt.Sprintf(`Row(%s>10)`, fldName), Remote: false, } @@ -743,7 +743,7 @@ func TestClient_ImportKeys(t *testing.T) { // Verify Range. queryRequest = &pilosa.QueryRequest{ - Query: fmt.Sprintf(`Range(%s>10)`, fldName), + Query: fmt.Sprintf(`Row(%s>10)`, fldName), Remote: false, } @@ -900,7 +900,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { t.Fatalf("unexpected existence columns: %+v", a) } }) @@ -935,7 +935,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected existence columns: %+v", a) } }) diff --git a/http/handler.go b/http/handler.go index d728bfcb5..b85be6b0e 100644 --- a/http/handler.go +++ b/http/handler.go @@ -24,8 +24,8 @@ import ( "io/ioutil" "net" "net/http" - _ "net/http/pprof" - "net/url" // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. + "net/url" "reflect" "runtime/debug" "strconv" @@ -192,12 +192,14 @@ func (h *Handler) populateValidators() { h.validators["PostClusterMessage"] = queryValidationSpecRequired() h.validators["GetFragmentBlockData"] = queryValidationSpecRequired() h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard") h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired() h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired() h.validators["GetNodes"] = queryValidationSpecRequired() h.validators["GetShardMax"] = queryValidationSpecRequired() h.validators["GetTranslateData"] = queryValidationSpecRequired("offset") + h.validators["PostTranslateKeys"] = queryValidationSpecRequired() } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -261,6 +263,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage") router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData") router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff") router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff") @@ -268,6 +271,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData") + router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys") router.Use(handler.queryArgValidator) router.Use(handler.extractTracing) @@ -1212,6 +1216,27 @@ type getFragmentBlocksResponse struct { Blocks []pilosa.FragmentBlock `json:"blocks"` } +// handleGetFragmentData handles GET /internal/fragment/data requests. +func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { + // Read shard parameter. + q := r.URL.Query() + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) + if err != nil { + http.Error(w, "shard required", http.StatusBadRequest) + return + } + // Retrieve fragment data from holder. + f, err := h.api.FragmentData(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + // Stream fragment to response body. + if _, err := f.WriteTo(w); err != nil { + h.logger.Printf("error streaming fragment data: %s", err) + } +} + // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { @@ -1401,7 +1426,11 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques type defaultClusterMessageResponse struct{} // translateStoreBufferSize is the buffer size used for streaming data. -const translateStoreBufferSize = 65536 +const translateStoreBufferSize = 1 << 16 // 64k + +// translateStoreBufferSizeMax is the maximum size that the buffer is allowed +// to grow before raising an error. +const translateStoreBufferSizeMax = 1 << 22 // 4Mb func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -1424,16 +1453,30 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) } // Copy from reader to client until store or client disconnect. - buf := make([]byte, translateStoreBufferSize) + useBufferSize := translateStoreBufferSize + buf := make([]byte, useBufferSize) for { // Read from store. n, err := rdr.Read(buf) if err == io.EOF { return + } else if err == pilosa.ErrTranslateReadTargetUndersized { + // Increase the buffer size and try to read again. + useBufferSize *= 2 + // Prevent the buffer from growing without bound. + if useBufferSize > translateStoreBufferSizeMax { + h.logger.Printf("http: translate store buffer exceeded max size: %s", err) + return + } + buf = make([]byte, useBufferSize) + continue } else if err != nil { h.logger.Printf("http: translate store read error: %s", err) return } else if n == 0 { + // Reset the default buffer size. + useBufferSize = translateStoreBufferSize + buf = make([]byte, useBufferSize) continue } @@ -1570,3 +1613,25 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request h.logger.Printf("writing import-roaring response: %v", err) } } + +func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + buf, err := h.api.TranslateKeys(r.Body) + if err != nil { + http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError) + } + + // Write response. + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing translate keys response: %v", err) + } +} diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index 7ca9c0401..ecbf8ff87 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -57,7 +57,6 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json) } } - } } @@ -93,7 +92,6 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) } } - } } @@ -178,6 +176,5 @@ func TestFieldOptionValidation(t *testing.T) { t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) } } - } } diff --git a/internal/clustertests/docker-compose-replication2.yml b/internal/clustertests/docker-compose-replication2.yml new file mode 100644 index 000000000..291699aee --- /dev/null +++ b/internal/clustertests/docker-compose-replication2.yml @@ -0,0 +1,58 @@ +version: '2' +services: + pilosa1: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33455:10101" + environment: + - PILOSA_CLUSTER_COORDINATOR=true + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa1:10101" + pilosa2: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33456:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa2:10101" + pilosa3: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33457:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa3:10101" + client1: + build: + context: . + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 + networks: + - pilosanet + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" +networks: + pilosanet: diff --git a/internal/private.pb.go b/internal/private.pb.go index 0c5862b13..c5a51741b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{0} + return fileDescriptor_private_8095a89af06a70de, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -93,7 +93,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{1} + return fileDescriptor_private_8095a89af06a70de, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -189,7 +189,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{2} + return fileDescriptor_private_8095a89af06a70de, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -240,7 +240,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{3} + return fileDescriptor_private_8095a89af06a70de, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -316,7 +316,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{4} + return fileDescriptor_private_8095a89af06a70de, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -370,7 +370,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{5} + return fileDescriptor_private_8095a89af06a70de, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -417,7 +417,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{6} + return fileDescriptor_private_8095a89af06a70de, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -466,7 +466,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{7} + return fileDescriptor_private_8095a89af06a70de, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -527,7 +527,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{8} + return fileDescriptor_private_8095a89af06a70de, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -575,7 +575,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{9} + return fileDescriptor_private_8095a89af06a70de, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -631,7 +631,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{10} + return fileDescriptor_private_8095a89af06a70de, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -693,7 +693,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{11} + return fileDescriptor_private_8095a89af06a70de, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -749,7 +749,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{12} + return fileDescriptor_private_8095a89af06a70de, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -812,7 +812,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{13} + return fileDescriptor_private_8095a89af06a70de, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -873,7 +873,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{14} + return fileDescriptor_private_8095a89af06a70de, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -921,7 +921,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{15} + return fileDescriptor_private_8095a89af06a70de, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -977,7 +977,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{16} + return fileDescriptor_private_8095a89af06a70de, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1041,7 +1041,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{17} + return fileDescriptor_private_8095a89af06a70de, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1110,7 +1110,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{18} + return fileDescriptor_private_8095a89af06a70de, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1165,7 +1165,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{19} + return fileDescriptor_private_8095a89af06a70de, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1221,7 +1221,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{20} + return fileDescriptor_private_8095a89af06a70de, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1283,7 +1283,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{21} + return fileDescriptor_private_8095a89af06a70de, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1338,7 +1338,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{22} + return fileDescriptor_private_8095a89af06a70de, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1394,7 +1394,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{23} + return fileDescriptor_private_8095a89af06a70de, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1458,7 +1458,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{24} + return fileDescriptor_private_8095a89af06a70de, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1528,7 +1528,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{25} + return fileDescriptor_private_8095a89af06a70de, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1591,7 +1591,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{26} + return fileDescriptor_private_8095a89af06a70de, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1646,7 +1646,7 @@ type ResizeInstruction struct { Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1657,7 +1657,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{27} + return fileDescriptor_private_8095a89af06a70de, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1714,9 +1714,9 @@ func (m *ResizeInstruction) GetSources() []*ResizeSource { return nil } -func (m *ResizeInstruction) GetSchema() *Schema { +func (m *ResizeInstruction) GetNodeStatus() *NodeStatus { if m != nil { - return m.Schema + return m.NodeStatus } return nil } @@ -1743,7 +1743,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{28} + return fileDescriptor_private_8095a89af06a70de, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1820,7 +1820,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{29} + return fileDescriptor_private_8095a89af06a70de, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1881,7 +1881,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{30} + return fileDescriptor_private_8095a89af06a70de, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1928,7 +1928,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{31} + return fileDescriptor_private_8095a89af06a70de, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1976,7 +1976,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{32} + return fileDescriptor_private_8095a89af06a70de, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2029,7 +2029,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{33} + return fileDescriptor_private_8095a89af06a70de, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3270,21 +3270,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.Schema != nil { - dAtA[i] = 0x2a + if m.ClusterStatus != nil { + dAtA[i] = 0x32 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n18 } - if m.ClusterStatus != nil { - dAtA[i] = 0x32 + if m.NodeStatus != nil { + dAtA[i] = 0x3a i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.NodeStatus.Size())) + n19, err := m.NodeStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -4169,14 +4169,14 @@ func (m *ResizeInstruction) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.Schema != nil { - l = m.Schema.Size() - n += 1 + l + sovPrivate(uint64(l)) - } if m.ClusterStatus != nil { l = m.ClusterStatus.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.NodeStatus != nil { + l = m.NodeStatus.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8034,39 +8034,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Schema == nil { - m.Schema = &Schema{} - } - if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterStatus", wireType) @@ -8100,6 +8067,39 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeStatus", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.NodeStatus == nil { + m.NodeStatus = &NodeStatus{} + } + if err := m.NodeStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8877,79 +8877,80 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_08d4c0c27f7a355f) } +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_8095a89af06a70de) } -var fileDescriptor_private_08d4c0c27f7a355f = []byte{ - // 1131 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5, - 0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51, - 0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c, - 0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7, - 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe2, 0x92, 0x47, 0xa8, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xb3, 0xbb, - 0x76, 0x1c, 0x12, 0x05, 0xee, 0xe6, 0xfc, 0xce, 0x99, 0xf3, 0x7d, 0xce, 0xec, 0x42, 0x77, 0x9c, - 0xc5, 0x27, 0x4c, 0xf2, 0x8d, 0x71, 0x26, 0xa4, 0x08, 0x5a, 0x71, 0x2a, 0x79, 0x96, 0xb2, 0x84, - 0x3c, 0x87, 0x76, 0x94, 0x0e, 0xf9, 0xd9, 0x0e, 0x97, 0x2c, 0x08, 0xc0, 0x7f, 0xc1, 0x27, 0x79, - 0xe8, 0xad, 0x39, 0xeb, 0x2d, 0x8a, 0xe7, 0xe0, 0x03, 0x58, 0xdc, 0xcb, 0xd8, 0xe0, 0x78, 0xfb, - 0x2c, 0xce, 0x25, 0x4f, 0x07, 0x3c, 0xf4, 0x91, 0x3b, 0x83, 0x92, 0x37, 0x0e, 0xdc, 0xfa, 0x2a, - 0xe6, 0xc9, 0xf0, 0xbb, 0xb1, 0x8c, 0x45, 0x9a, 0x07, 0xef, 0x42, 0x7b, 0x8b, 0x0d, 0x8e, 0xf8, - 0xde, 0x64, 0xcc, 0x51, 0x63, 0x9b, 0x56, 0x40, 0xc9, 0xed, 0xc7, 0xaf, 0xb5, 0xc6, 0x2e, 0xad, - 0x80, 0x60, 0x0d, 0x3a, 0x7b, 0xf1, 0x88, 0x7f, 0x5f, 0xb0, 0x54, 0x16, 0xa3, 0xb0, 0x81, 0xb7, - 0xeb, 0x90, 0x72, 0x15, 0x15, 0xb7, 0x90, 0x85, 0xe7, 0x60, 0x09, 0xbc, 0x9d, 0x38, 0x0d, 0xdb, - 0x6b, 0xce, 0xba, 0x47, 0xd5, 0x11, 0x11, 0x76, 0x16, 0x82, 0x41, 0xd8, 0x59, 0x19, 0x62, 0x67, - 0x3a, 0xc4, 0x5d, 0xd1, 0x97, 0x2c, 0x1d, 0xb2, 0x6c, 0x78, 0x10, 0xf3, 0xd3, 0xf0, 0x96, 0x0e, - 0x71, 0x1a, 0x25, 0x04, 0x16, 0xa3, 0xd1, 0x58, 0x64, 0x92, 0xf2, 0x7c, 0x2c, 0xd2, 0x1c, 0x2d, - 0x6e, 0x67, 0x59, 0xe8, 0xa0, 0x13, 0xea, 0x48, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0x0c, 0x8e, 0x7b, - 0x4c, 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0xff, 0x43, 0x03, 0x73, 0x6c, 0xe4, 0x34, 0xa1, - 0x50, 0xcc, 0x57, 0xe8, 0x6a, 0x14, 0x09, 0x85, 0xe2, 0x7d, 0xcc, 0x98, 0x4f, 0x35, 0xa1, 0xd0, - 0xfe, 0x11, 0xcb, 0x86, 0x98, 0x29, 0x9f, 0x6a, 0x42, 0xc5, 0x82, 0xde, 0xea, 0xf4, 0xe0, 0x99, - 0x44, 0xb0, 0x5c, 0xb3, 0x6f, 0xdc, 0x5c, 0x81, 0x26, 0x15, 0xa7, 0x51, 0x2f, 0x0f, 0x9d, 0x35, - 0x6f, 0xdd, 0xa7, 0x86, 0xc2, 0x22, 0x88, 0xa4, 0x18, 0xa5, 0x8a, 0xe5, 0x22, 0xab, 0x02, 0xc8, - 0x3d, 0x68, 0x60, 0x45, 0x54, 0x94, 0xd5, 0x5d, 0x75, 0x24, 0x3f, 0x3b, 0xd0, 0xde, 0x61, 0x67, - 0xe8, 0x46, 0x1e, 0x3c, 0x85, 0x96, 0xcd, 0x13, 0x0a, 0x75, 0x1e, 0xbf, 0xbf, 0x61, 0x1b, 0x6c, - 0xa3, 0x14, 0xdb, 0xb0, 0x32, 0xdb, 0xa9, 0xcc, 0x26, 0xb4, 0xbc, 0xf2, 0xf6, 0xe7, 0xd0, 0x9d, - 0x62, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, - 0x73, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x03, 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, - 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x79, 0xc6, 0x75, 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, - 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, 0x37, 0xd3, 0xf1, 0x0f, 0x7a, 0x49, - 0xdf, 0xfa, 0x70, 0xb5, 0x6c, 0xf0, 0x00, 0x7c, 0x35, 0x6a, 0xe8, 0x42, 0xe7, 0xf1, 0x9d, 0x2a, - 0x4f, 0xe5, 0x14, 0x52, 0x14, 0x20, 0x89, 0x55, 0x8a, 0xfe, 0x5c, 0x19, 0xd8, 0x9c, 0x56, 0x7a, - 0x68, 0x4c, 0x79, 0x68, 0x6a, 0xa5, 0x32, 0x55, 0x1f, 0x53, 0x63, 0xed, 0x99, 0x0d, 0xf7, 0xa6, - 0xd6, 0xc8, 0x00, 0xde, 0xd1, 0x1a, 0xbe, 0x3c, 0x61, 0x71, 0xc2, 0x0e, 0x93, 0x6b, 0x56, 0x64, - 0x8e, 0xe3, 0x21, 0x2c, 0xe0, 0xdd, 0xa8, 0x67, 0xa6, 0xc0, 0x92, 0xe4, 0x07, 0x23, 0xaf, 0x5a, - 0x7f, 0x97, 0x8d, 0xb8, 0xd1, 0x86, 0xe7, 0x32, 0x5e, 0xf7, 0xea, 0x78, 0x95, 0x61, 0x35, 0x2e, - 0x6a, 0xd5, 0x79, 0xca, 0x30, 0x12, 0xe4, 0x09, 0x34, 0xfb, 0x83, 0x23, 0x3e, 0x62, 0xc1, 0x87, - 0xb0, 0x80, 0x1e, 0xf2, 0xdc, 0x74, 0xf4, 0xed, 0x99, 0x4a, 0x51, 0xcb, 0x27, 0x3d, 0x13, 0xd9, - 0x5c, 0x9f, 0x1e, 0x40, 0x13, 0xad, 0xe7, 0xa1, 0x3f, 0xab, 0x06, 0x71, 0x6a, 0xd8, 0x64, 0x1b, - 0xbc, 0x7d, 0x1a, 0xa9, 0x49, 0x45, 0x0f, 0xac, 0x16, 0x43, 0x29, 0xdd, 0x5f, 0x8b, 0x5c, 0x9a, - 0x3c, 0xe1, 0x59, 0x61, 0x2f, 0x45, 0x26, 0x31, 0x47, 0x5d, 0x8a, 0x67, 0x92, 0x83, 0xbf, 0x2b, - 0x86, 0x3c, 0x58, 0x04, 0x37, 0xea, 0x19, 0x1d, 0x6e, 0xd4, 0x0b, 0xde, 0x43, 0xf5, 0x26, 0x35, - 0xdd, 0xca, 0x89, 0x7d, 0x1a, 0x51, 0x34, 0x7c, 0x1f, 0xba, 0x51, 0xbe, 0x25, 0x44, 0x36, 0x8c, - 0x53, 0x26, 0x45, 0x66, 0xde, 0x80, 0x69, 0x10, 0x27, 0x48, 0x32, 0xa9, 0x37, 0x76, 0x9b, 0x6a, - 0x82, 0x3c, 0x83, 0x25, 0x65, 0x14, 0x09, 0x5b, 0xef, 0x15, 0x68, 0x2a, 0xac, 0x74, 0xc2, 0x50, - 0x95, 0x06, 0xb7, 0xae, 0xe1, 0x5b, 0xad, 0x61, 0xfb, 0x84, 0xa7, 0xb2, 0xd6, 0x31, 0x48, 0xa3, - 0x82, 0x2e, 0xd5, 0x44, 0x40, 0x74, 0x80, 0x26, 0x92, 0xc5, 0x2a, 0x12, 0x85, 0x52, 0xe4, 0x91, - 0x5f, 0x1d, 0x00, 0xeb, 0x50, 0x91, 0x97, 0x57, 0x9c, 0xcb, 0xaf, 0x04, 0xeb, 0xb6, 0xf2, 0x66, - 0x5a, 0x96, 0x2a, 0x29, 0x8d, 0x53, 0xdb, 0x19, 0x1f, 0x57, 0x9d, 0xa1, 0x4b, 0x7a, 0x77, 0xa6, - 0x33, 0xb4, 0xd5, 0xaa, 0x3f, 0x5e, 0x42, 0xa7, 0x86, 0xcf, 0xed, 0x92, 0x8f, 0xca, 0x2e, 0x71, - 0x67, 0x55, 0x22, 0x6e, 0x54, 0xda, 0x5e, 0x79, 0x01, 0x9d, 0x1a, 0x3c, 0x57, 0xe3, 0x3a, 0xdc, - 0x9e, 0x9e, 0x43, 0xbb, 0xdf, 0x67, 0x61, 0x12, 0x43, 0x77, 0x2b, 0x29, 0x72, 0xc9, 0x33, 0xa3, - 0x4e, 0x3d, 0x0a, 0x1a, 0x28, 0x8b, 0x57, 0x01, 0xf3, 0xeb, 0x17, 0xdc, 0x87, 0x86, 0x4a, 0xa3, - 0x1e, 0xa7, 0x8b, 0x39, 0xd6, 0x4c, 0x72, 0x00, 0xad, 0xcd, 0x7e, 0xf4, 0x3c, 0x13, 0xc5, 0x78, - 0xae, 0xd3, 0xf6, 0x4d, 0x77, 0x2f, 0xbe, 0xe9, 0xde, 0x85, 0x37, 0xdd, 0x2f, 0xdf, 0x74, 0xd2, - 0x87, 0x65, 0xbd, 0x2a, 0xd5, 0x14, 0xdf, 0x64, 0xe1, 0xd8, 0x87, 0xd4, 0xab, 0x3d, 0xa4, 0x7d, - 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x54, 0xfa, 0xbb, 0x0b, 0xcb, 0x94, 0xe7, 0xf1, 0x6b, 0x1e, 0xa5, - 0xb9, 0xcc, 0x8a, 0x81, 0xda, 0x49, 0xea, 0xfe, 0x37, 0xe2, 0xd0, 0x64, 0xdb, 0xa3, 0x9a, 0xb8, - 0x4e, 0xa7, 0x07, 0x8f, 0xa0, 0x33, 0x3b, 0xb3, 0x17, 0x45, 0xeb, 0x22, 0xc1, 0x23, 0x58, 0xe8, - 0x8b, 0x22, 0x1b, 0x94, 0xed, 0x5b, 0xdb, 0x93, 0xda, 0x33, 0xcd, 0xa6, 0x56, 0xac, 0x36, 0x1a, - 0x8d, 0x2b, 0x46, 0xe3, 0xe9, 0x4c, 0x2b, 0x85, 0x4d, 0xbc, 0xf0, 0x56, 0x75, 0x61, 0x8a, 0x4d, - 0xa7, 0xa5, 0xc9, 0x2f, 0x0e, 0xdc, 0xaa, 0xbb, 0x70, 0xad, 0xc1, 0x2d, 0x2b, 0xe2, 0xce, 0xad, - 0x88, 0x37, 0xaf, 0x22, 0x7e, 0x55, 0x91, 0xea, 0x9b, 0xa0, 0x51, 0xfb, 0x26, 0x20, 0xc7, 0x70, - 0xef, 0x42, 0x99, 0xb6, 0xc4, 0x68, 0xac, 0xfa, 0xe1, 0x5f, 0x94, 0x4b, 0xad, 0xb4, 0x2c, 0x33, - 0x85, 0x6a, 0x53, 0x4d, 0x90, 0x4f, 0xe1, 0x6e, 0x9f, 0xcb, 0x5a, 0x91, 0x6c, 0xb7, 0xad, 0x81, - 0xb7, 0xcb, 0x4f, 0x2f, 0x09, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xee, 0x8f, 0x87, 0x4c, 0xf2, 0x1b, - 0xdd, 0xde, 0x84, 0xd6, 0x9e, 0x18, 0x8b, 0x44, 0xbc, 0x9a, 0x5c, 0x31, 0xf5, 0x21, 0x2c, 0xe8, - 0xfd, 0xad, 0xd7, 0x48, 0x9b, 0x5a, 0x92, 0xdc, 0x51, 0x0d, 0x3d, 0x60, 0xc9, 0xa0, 0x48, 0x94, - 0x1b, 0xea, 0x7b, 0x31, 0xdf, 0x5c, 0xfa, 0xe3, 0x7c, 0xd5, 0xf9, 0xf3, 0x7c, 0xd5, 0x79, 0x73, - 0xbe, 0xea, 0xfc, 0xf6, 0xd7, 0xea, 0xff, 0x0e, 0x9b, 0xf8, 0xdf, 0xf1, 0xe4, 0xef, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x61, 0x80, 0xe4, 0xef, 0x88, 0x0c, 0x00, 0x00, +var fileDescriptor_private_8095a89af06a70de = []byte{ + // 1139 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0xc5, + 0x1b, 0xff, 0xef, 0x21, 0x8e, 0xfd, 0x39, 0x4e, 0x93, 0x6d, 0x9b, 0xff, 0x16, 0x50, 0x08, 0xa3, + 0x8a, 0x86, 0x4a, 0x84, 0xaa, 0xe5, 0x82, 0x53, 0xa5, 0x92, 0x38, 0x94, 0xa5, 0x24, 0x94, 0x71, + 0x92, 0x3b, 0x2e, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0x63, 0x76, 0x67, 0x93, 0xb8, 0x17, 0xdc, + 0x82, 0xc4, 0x0b, 0xf0, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x42, 0x15, 0x5e, 0x04, 0xcd, 0x37, 0x33, + 0xbb, 0x6b, 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xbe, 0xdf, 0x77, 0x3e, 0xae, 0x0d, 0x9d, 0x51, 0x16, + 0x9f, 0x30, 0xc9, 0x37, 0x46, 0x99, 0x90, 0x22, 0x68, 0xc6, 0xa9, 0xe4, 0x59, 0xca, 0x12, 0xf2, + 0x1c, 0x5a, 0x51, 0x3a, 0xe0, 0x67, 0x3b, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x05, 0x1f, 0xe7, 0xa1, + 0xb7, 0xe6, 0xac, 0x37, 0x29, 0xbe, 0x83, 0xf7, 0x61, 0x71, 0x2f, 0x63, 0xfd, 0xe3, 0xed, 0xb3, + 0x38, 0x97, 0x3c, 0xed, 0xf3, 0xd0, 0x47, 0xee, 0x14, 0x4a, 0xde, 0x38, 0xb0, 0xf0, 0x55, 0xcc, + 0x93, 0xc1, 0x77, 0x23, 0x19, 0x8b, 0x34, 0x0f, 0xde, 0x81, 0xd6, 0x16, 0xeb, 0x1f, 0xf1, 0xbd, + 0xf1, 0x88, 0xa3, 0xc5, 0x16, 0xad, 0x80, 0x92, 0xdb, 0x8b, 0x5f, 0x6b, 0x8b, 0x1d, 0x5a, 0x01, + 0xc1, 0x1a, 0xb4, 0xf7, 0xe2, 0x21, 0xff, 0xbe, 0x60, 0xa9, 0x2c, 0x86, 0xe1, 0x1c, 0x6a, 0xd7, + 0x21, 0x15, 0x2a, 0x1a, 0x6e, 0x22, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0x3b, 0x71, 0x1a, 0xb6, 0xd6, + 0x9c, 0x75, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x2c, 0x04, 0x83, 0xb0, 0xb3, 0x32, 0xc5, 0xf6, 0x64, + 0x8a, 0xbb, 0xa2, 0x27, 0x59, 0x3a, 0x60, 0xd9, 0xe0, 0x20, 0xe6, 0xa7, 0xe1, 0x82, 0x4e, 0x71, + 0x12, 0x25, 0x04, 0x16, 0xa3, 0xe1, 0x48, 0x64, 0x92, 0xf2, 0x7c, 0x24, 0xd2, 0x1c, 0x3d, 0x6e, + 0x67, 0x59, 0xe8, 0x60, 0x10, 0xea, 0x49, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0xf4, 0x8f, 0xbb, 0x4c, + 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0x3b, 0x30, 0x87, 0x35, 0x36, 0x72, 0x9a, 0x50, 0x28, + 0xd6, 0x2b, 0x74, 0x35, 0x8a, 0x84, 0x42, 0x51, 0x1f, 0x2b, 0xe6, 0x53, 0x4d, 0x28, 0xb4, 0x77, + 0xc4, 0xb2, 0x01, 0x56, 0xca, 0xa7, 0x9a, 0x50, 0xb9, 0x60, 0xb4, 0xba, 0x3c, 0xf8, 0x26, 0x11, + 0x2c, 0xd7, 0xfc, 0x9b, 0x30, 0x57, 0xa0, 0x41, 0xc5, 0x69, 0xd4, 0xcd, 0x43, 0x67, 0xcd, 0x5b, + 0xf7, 0xa9, 0xa1, 0xb0, 0x09, 0x22, 0x29, 0x86, 0xa9, 0x62, 0xb9, 0xc8, 0xaa, 0x00, 0x72, 0x0f, + 0xe6, 0xb0, 0x23, 0x2a, 0xcb, 0x4a, 0x57, 0x3d, 0xc9, 0xcf, 0x0e, 0xb4, 0x76, 0xd8, 0x19, 0x86, + 0x91, 0x07, 0x4f, 0xa1, 0x69, 0xeb, 0x84, 0x42, 0xed, 0xc7, 0xef, 0x6d, 0xd8, 0x01, 0xdb, 0x28, + 0xc5, 0x36, 0xac, 0xcc, 0x76, 0x2a, 0xb3, 0x31, 0x2d, 0x55, 0xde, 0xfa, 0x1c, 0x3a, 0x13, 0x2c, + 0xe5, 0xef, 0x98, 0x8f, 0x6d, 0x55, 0x8f, 0xf9, 0x58, 0xe5, 0x7f, 0xc2, 0x92, 0x82, 0x63, 0xad, + 0x7c, 0xaa, 0x89, 0xcf, 0xdc, 0x4f, 0x1c, 0x72, 0x00, 0xc1, 0x56, 0xc6, 0x99, 0xe4, 0xe8, 0x64, + 0x87, 0xe7, 0x39, 0x7b, 0xc5, 0x2f, 0xaf, 0xb8, 0xae, 0xa2, 0x5b, 0xaf, 0x62, 0xd9, 0x07, 0xaf, + 0xd6, 0x07, 0xf2, 0x10, 0x82, 0x2e, 0x4f, 0xb8, 0xe4, 0x66, 0x3b, 0xfe, 0xc1, 0x2e, 0xe9, 0xd9, + 0x18, 0xae, 0x96, 0x0d, 0x1e, 0x80, 0xaf, 0x56, 0x0d, 0x43, 0x68, 0x3f, 0xbe, 0x5d, 0xd5, 0xa9, + 0xdc, 0x42, 0x8a, 0x02, 0x24, 0xb1, 0x46, 0x31, 0x9e, 0x2b, 0x13, 0x9b, 0x31, 0x4a, 0x0f, 0x8d, + 0x2b, 0x0f, 0x5d, 0xad, 0x54, 0xae, 0xea, 0x6b, 0x6a, 0xbc, 0x3d, 0xb3, 0xe9, 0xde, 0xd4, 0x1b, + 0xe9, 0xc3, 0xdb, 0xda, 0xc2, 0x97, 0x27, 0x2c, 0x4e, 0xd8, 0x61, 0x72, 0xcd, 0x8e, 0xcc, 0x08, + 0x3c, 0x84, 0x79, 0xd4, 0x8d, 0xba, 0x66, 0x0b, 0x2c, 0x49, 0x7e, 0x30, 0xf2, 0x6a, 0xf4, 0x77, + 0xd9, 0x90, 0x1b, 0x6b, 0xf8, 0x2e, 0xf3, 0x75, 0xaf, 0xce, 0x57, 0x39, 0x56, 0xeb, 0xa2, 0x4e, + 0x9d, 0xa7, 0x1c, 0x23, 0x41, 0x9e, 0x40, 0xa3, 0xd7, 0x3f, 0xe2, 0x43, 0x16, 0x7c, 0x00, 0xf3, + 0x18, 0x21, 0xcf, 0xcd, 0x44, 0xdf, 0x9a, 0xea, 0x14, 0xb5, 0x7c, 0xd2, 0x35, 0x99, 0xcd, 0x8c, + 0xe9, 0x01, 0x34, 0xd0, 0x7b, 0x1e, 0xfa, 0xd3, 0x66, 0x10, 0xa7, 0x86, 0x4d, 0xb6, 0xc1, 0xdb, + 0xa7, 0x91, 0xda, 0x54, 0x8c, 0xc0, 0x5a, 0x31, 0x94, 0xb2, 0xfd, 0xb5, 0xc8, 0xa5, 0xa9, 0x13, + 0xbe, 0x15, 0xf6, 0x52, 0x64, 0x12, 0x6b, 0xd4, 0xa1, 0xf8, 0x26, 0x39, 0xf8, 0xbb, 0x62, 0xc0, + 0x83, 0x45, 0x70, 0xa3, 0xae, 0xb1, 0xe1, 0x46, 0xdd, 0xe0, 0x5d, 0x34, 0x6f, 0x4a, 0xd3, 0xa9, + 0x82, 0xd8, 0xa7, 0x11, 0x45, 0xc7, 0xf7, 0xa1, 0x13, 0xe5, 0x5b, 0x42, 0x64, 0x83, 0x38, 0x65, + 0x52, 0x64, 0xe6, 0x1b, 0x30, 0x09, 0xe2, 0x06, 0x49, 0x26, 0xf5, 0xc5, 0x6e, 0x51, 0x4d, 0x90, + 0x67, 0xb0, 0xa4, 0x9c, 0x22, 0x61, 0xfb, 0xbd, 0x02, 0x0d, 0x85, 0x95, 0x41, 0x18, 0xaa, 0xb2, + 0xe0, 0xd6, 0x2d, 0x7c, 0xab, 0x2d, 0x6c, 0x9f, 0xf0, 0x54, 0xd6, 0x26, 0x06, 0x69, 0x34, 0xd0, + 0xa1, 0x9a, 0x08, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x58, 0x65, 0xa2, 0x50, 0x8a, 0x3c, 0xf2, 0xab, + 0x03, 0x60, 0x03, 0x2a, 0xf2, 0x52, 0xc5, 0xb9, 0x5c, 0x25, 0x58, 0xb7, 0x9d, 0x37, 0xdb, 0xb2, + 0x54, 0x49, 0x69, 0x9c, 0xda, 0xc9, 0xf8, 0xa8, 0x9a, 0x0c, 0xdd, 0xd2, 0xbb, 0x53, 0x93, 0xa1, + 0xbd, 0x56, 0xf3, 0xf1, 0x12, 0xda, 0x35, 0x7c, 0xe6, 0x94, 0x7c, 0x58, 0x4e, 0x89, 0x3b, 0x6d, + 0x12, 0x71, 0x63, 0xd2, 0xce, 0xca, 0x0b, 0x68, 0xd7, 0xe0, 0x99, 0x16, 0xd7, 0xe1, 0xd6, 0xe4, + 0x1e, 0xda, 0xfb, 0x3e, 0x0d, 0x93, 0x18, 0x3a, 0x5b, 0x49, 0x91, 0x4b, 0x9e, 0x19, 0x73, 0xea, + 0xa3, 0xa0, 0x81, 0xb2, 0x79, 0x15, 0x30, 0xbb, 0x7f, 0xc1, 0x7d, 0x98, 0x53, 0x65, 0xd4, 0xeb, + 0x74, 0xb1, 0xc6, 0x9a, 0x49, 0x0e, 0xa0, 0xb9, 0xd9, 0x8b, 0x9e, 0x67, 0xa2, 0x18, 0xcd, 0x0c, + 0xda, 0x7e, 0xd3, 0xdd, 0x8b, 0xdf, 0x74, 0xef, 0xc2, 0x37, 0xdd, 0x2f, 0xbf, 0xe9, 0xa4, 0x07, + 0xcb, 0xfa, 0x54, 0xaa, 0x2d, 0xbe, 0xc9, 0xc1, 0xb1, 0x1f, 0x52, 0xaf, 0xf6, 0x21, 0xed, 0xc1, + 0xb2, 0xbe, 0x67, 0xff, 0xa5, 0xd1, 0xdf, 0x5d, 0x58, 0xa6, 0x3c, 0x8f, 0x5f, 0xf3, 0x28, 0xcd, + 0x65, 0x56, 0xf4, 0xd5, 0x4d, 0x52, 0xfa, 0xdf, 0x88, 0x43, 0x53, 0x6d, 0x8f, 0x6a, 0xe2, 0x3a, + 0x93, 0x1e, 0x3c, 0x82, 0xf6, 0xf4, 0xce, 0x5e, 0x14, 0xad, 0x8b, 0x04, 0x8f, 0x60, 0xbe, 0x27, + 0x8a, 0xac, 0x5f, 0x8e, 0x6f, 0xed, 0x4e, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xe0, 0xe9, 0xd4, + 0x80, 0x84, 0x0d, 0xf4, 0xf2, 0xff, 0x4a, 0x6f, 0x82, 0x4d, 0xa7, 0xc6, 0xe9, 0xe3, 0xfa, 0x2e, + 0x86, 0xf3, 0xa8, 0x7b, 0x67, 0x32, 0x42, 0xa3, 0x58, 0x93, 0x23, 0xbf, 0x38, 0xb0, 0x50, 0x0f, + 0xe7, 0x5a, 0x4b, 0x5c, 0x76, 0xc7, 0x9d, 0xd9, 0x1d, 0x6f, 0x56, 0x77, 0xfc, 0xaa, 0x3b, 0xd5, + 0xef, 0x83, 0xb9, 0xda, 0xef, 0x03, 0x72, 0x0c, 0xf7, 0x2e, 0xb4, 0x6c, 0x4b, 0x0c, 0x47, 0x6a, + 0x36, 0xfe, 0x45, 0xeb, 0xd4, 0x79, 0xcb, 0x32, 0xd3, 0xb4, 0x16, 0xd5, 0x04, 0xf9, 0x14, 0xee, + 0xf6, 0xb8, 0xac, 0x35, 0xcc, 0x4e, 0xde, 0x1a, 0x78, 0xbb, 0xfc, 0xf4, 0x92, 0xf4, 0x15, 0x8b, + 0x7c, 0x01, 0xe1, 0xfe, 0x68, 0xc0, 0x24, 0xbf, 0x91, 0xf6, 0x26, 0x34, 0xf7, 0xc4, 0x48, 0x24, + 0xe2, 0xd5, 0xf8, 0x8a, 0x0b, 0x10, 0xc2, 0xbc, 0xbe, 0xe5, 0xfa, 0xa4, 0xb4, 0xa8, 0x25, 0xc9, + 0x6d, 0x35, 0xdc, 0x7d, 0x96, 0xf4, 0x8b, 0x44, 0x85, 0xa1, 0x7e, 0x3b, 0xe6, 0x9b, 0x4b, 0x7f, + 0x9c, 0xaf, 0x3a, 0x7f, 0x9e, 0xaf, 0x3a, 0x6f, 0xce, 0x57, 0x9d, 0xdf, 0xfe, 0x5a, 0xfd, 0xdf, + 0x61, 0x03, 0xff, 0x83, 0x3c, 0xf9, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xd8, 0x6d, 0x1f, 0x94, + 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1cd88c82c..971bb5f69 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -159,7 +159,7 @@ message ResizeInstruction { Node Node = 2; Node Coordinator = 3; repeated ResizeSource Sources = 4; - Schema Schema = 5; + NodeStatus NodeStatus = 7; ClusterStatus ClusterStatus = 6; } diff --git a/internal/public.pb.go b/internal/public.pb.go index 34ac4db9f..5cd86a833 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{0} + return fileDescriptor_public_f65cfea24ac19f54, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{1} + return fileDescriptor_public_f65cfea24ac19f54, []int{1} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{2} + return fileDescriptor_public_f65cfea24ac19f54, []int{2} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -206,6 +206,7 @@ func (m *Pair) GetCount() uint64 { type FieldRow struct { Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` + RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -215,7 +216,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{3} + return fileDescriptor_public_f65cfea24ac19f54, []int{3} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -258,6 +259,13 @@ func (m *FieldRow) GetRowID() uint64 { return 0 } +func (m *FieldRow) GetRowKey() string { + if m != nil { + return m.RowKey + } + return "" +} + type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` @@ -270,7 +278,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{4} + return fileDescriptor_public_f65cfea24ac19f54, []int{4} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,7 +333,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{5} + return fileDescriptor_public_f65cfea24ac19f54, []int{5} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -368,69 +376,6 @@ func (m *ValCount) GetCount() int64 { return 0 } -type Bit struct { - RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` - ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` - Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Bit) Reset() { *m = Bit{} } -func (m *Bit) String() string { return proto.CompactTextString(m) } -func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{6} -} -func (m *Bit) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Bit) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Bit.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Bit) XXX_Merge(src proto.Message) { - xxx_messageInfo_Bit.Merge(dst, src) -} -func (m *Bit) XXX_Size() int { - return m.Size() -} -func (m *Bit) XXX_DiscardUnknown() { - xxx_messageInfo_Bit.DiscardUnknown(m) -} - -var xxx_messageInfo_Bit proto.InternalMessageInfo - -func (m *Bit) GetRowID() uint64 { - if m != nil { - return m.RowID - } - return 0 -} - -func (m *Bit) GetColumnID() uint64 { - if m != nil { - return m.ColumnID - } - return 0 -} - -func (m *Bit) GetTimestamp() int64 { - if m != nil { - return m.Timestamp - } - return 0 -} - type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` @@ -444,7 +389,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{7} + return fileDescriptor_public_f65cfea24ac19f54, []int{6} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +455,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{8} + return fileDescriptor_public_f65cfea24ac19f54, []int{7} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +537,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{9} + return fileDescriptor_public_f65cfea24ac19f54, []int{8} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -644,7 +589,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{10} + return fileDescriptor_public_f65cfea24ac19f54, []int{9} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,7 +673,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{11} + return fileDescriptor_public_f65cfea24ac19f54, []int{10} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -797,7 +742,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{12} + return fileDescriptor_public_f65cfea24ac19f54, []int{11} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,7 +852,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{13} + return fileDescriptor_public_f65cfea24ac19f54, []int{12} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1008,7 +953,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{14} + return fileDescriptor_public_f65cfea24ac19f54, []int{13} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1079,6 +1024,116 @@ func (m *ImportValueRequest) GetValues() []int64 { return nil } +type TranslateKeysRequest struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } +func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysRequest) ProtoMessage() {} +func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_public_f65cfea24ac19f54, []int{14} +} +func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) +} +func (m *TranslateKeysRequest) XXX_Size() int { + return m.Size() +} +func (m *TranslateKeysRequest) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateKeysRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateKeysRequest proto.InternalMessageInfo + +func (m *TranslateKeysRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *TranslateKeysRequest) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *TranslateKeysRequest) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + +type TranslateKeysResponse struct { + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } +func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysResponse) ProtoMessage() {} +func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_public_f65cfea24ac19f54, []int{15} +} +func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) +} +func (m *TranslateKeysResponse) XXX_Size() int { + return m.Size() +} +func (m *TranslateKeysResponse) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateKeysResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateKeysResponse proto.InternalMessageInfo + +func (m *TranslateKeysResponse) GetIDs() []uint64 { + if m != nil { + return m.IDs + } + return nil +} + type ImportRoaringRequestView struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` @@ -1091,7 +1146,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{15} + return fileDescriptor_public_f65cfea24ac19f54, []int{16} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1146,7 +1201,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_5eafd62083455670, []int{16} + return fileDescriptor_public_f65cfea24ac19f54, []int{17} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1196,7 +1251,6 @@ func init() { proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") - proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") proto.RegisterType((*AttrMap)(nil), "internal.AttrMap") @@ -1205,6 +1259,8 @@ func init() { proto.RegisterType((*QueryResult)(nil), "internal.QueryResult") proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") + proto.RegisterType((*TranslateKeysRequest)(nil), "internal.TranslateKeysRequest") + proto.RegisterType((*TranslateKeysResponse)(nil), "internal.TranslateKeysResponse") proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") } @@ -1389,6 +1445,12 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) } + if len(m.RowKey) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) + i += copy(dAtA[i:], m.RowKey) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -1464,42 +1526,6 @@ func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Bit) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Bit) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.RowID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) - } - if m.ColumnID != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID)) - } - if m.Timestamp != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Timestamp)) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2091,6 +2117,92 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.IDs) > 0 { + dAtA23 := make([]byte, len(m.IDs)*10) + var j22 int + for _, num := range m.IDs { + for num >= 1<<7 { + dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j22++ + } + dAtA23[j22] = uint8(num) + j22++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j22)) + i += copy(dAtA[i:], dAtA23[:j22]) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2267,6 +2379,10 @@ func (m *FieldRow) Size() (n int) { if m.RowID != 0 { n += 1 + sovPublic(uint64(m.RowID)) } + l = len(m.RowKey) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -2312,27 +2428,6 @@ func (m *ValCount) Size() (n int) { return n } -func (m *Bit) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.RowID != 0 { - n += 1 + sovPublic(uint64(m.RowID)) - } - if m.ColumnID != 0 { - n += 1 + sovPublic(uint64(m.ColumnID)) - } - if m.Timestamp != 0 { - n += 1 + sovPublic(uint64(m.Timestamp)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - func (m *ColumnAttrSet) Size() (n int) { if m == nil { return 0 @@ -2622,6 +2717,51 @@ func (m *ImportValueRequest) Size() (n int) { return n } +func (m *TranslateKeysRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *TranslateKeysResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.IDs) > 0 { + l = 0 + for _, e := range m.IDs { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *ImportRoaringRequestView) Size() (n int) { if m == nil { return 0 @@ -3208,6 +3348,35 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowKey", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowKey = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3420,114 +3589,6 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } return nil } -func (m *Bit) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Bit: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Bit: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) - } - m.RowID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.RowID |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType) - } - m.ColumnID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ColumnID |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) - } - m.Timestamp = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Timestamp |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipPublic(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5344,6 +5405,268 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateKeysRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateKeysRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateKeysResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateKeysResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 3: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.IDs) == 0 { + m.IDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5662,63 +5985,63 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_5eafd62083455670) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_f65cfea24ac19f54) } -var fileDescriptor_public_5eafd62083455670 = []byte{ - // 870 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x8e, 0xdb, 0x44, - 0x14, 0x66, 0x62, 0x27, 0x71, 0x4e, 0x36, 0xa1, 0x1a, 0x2d, 0xc5, 0x42, 0x55, 0xb0, 0x2c, 0x84, - 0x7c, 0xb5, 0x95, 0x82, 0x54, 0xf5, 0x8a, 0x9f, 0x6d, 0xb6, 0x28, 0x2a, 0xac, 0xe0, 0x6c, 0x09, - 0xe2, 0xd2, 0x6d, 0xa6, 0xad, 0x25, 0xc7, 0x63, 0xec, 0x31, 0x69, 0x9e, 0x83, 0x1b, 0x1e, 0x81, - 0x0b, 0x1e, 0xa4, 0x97, 0x88, 0x27, 0x80, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0x64, 0x9c, 0x6c, 0x59, - 0x71, 0xc1, 0xdd, 0x7c, 0xe7, 0xcc, 0x39, 0xfe, 0xbe, 0x39, 0x3f, 0x09, 0x9c, 0x94, 0xcd, 0xb3, - 0x3c, 0x7b, 0x7e, 0x56, 0x56, 0x52, 0x49, 0x1e, 0x64, 0x85, 0x12, 0x55, 0x91, 0xe6, 0xf1, 0x0f, - 0xe0, 0xa1, 0xdc, 0xf2, 0x10, 0x86, 0x8f, 0x64, 0xde, 0x6c, 0x8a, 0x3a, 0x64, 0x91, 0x97, 0xf8, - 0x68, 0x21, 0xff, 0x08, 0xfa, 0x5f, 0x28, 0x55, 0xd5, 0x61, 0x2f, 0xf2, 0x92, 0xf1, 0x7c, 0x7a, - 0x66, 0x43, 0xcf, 0xb4, 0x19, 0x8d, 0x93, 0x73, 0xf0, 0x9f, 0x88, 0x5d, 0x1d, 0x7a, 0x91, 0x97, - 0x8c, 0x90, 0xce, 0xf1, 0x43, 0x98, 0xa2, 0xdc, 0x2e, 0xd7, 0xa2, 0x50, 0xd9, 0x8b, 0x4c, 0x98, - 0x5b, 0x28, 0xb7, 0xf6, 0x13, 0x74, 0xde, 0x47, 0xf6, 0x3a, 0x91, 0x9f, 0x82, 0xff, 0x4d, 0x9a, - 0x55, 0x7c, 0x0a, 0xbd, 0xe5, 0x22, 0x64, 0x11, 0x4b, 0x7c, 0xec, 0x2d, 0x17, 0xfc, 0x14, 0xfa, - 0x8f, 0x64, 0x53, 0xa8, 0xb0, 0x47, 0x26, 0x03, 0xf8, 0x1d, 0xf0, 0x9e, 0x88, 0x5d, 0xe8, 0x45, - 0x2c, 0x19, 0xa1, 0x3e, 0xc6, 0x0f, 0x20, 0x78, 0x9c, 0x89, 0x7c, 0xad, 0x95, 0x9d, 0x42, 0x9f, - 0xce, 0x94, 0x66, 0x84, 0x06, 0x68, 0xab, 0xe6, 0xb6, 0xb0, 0x99, 0x08, 0xc4, 0x5f, 0x01, 0x7c, - 0x59, 0xc9, 0xa6, 0x34, 0x79, 0x13, 0xe8, 0x13, 0x22, 0xba, 0xe3, 0x39, 0x77, 0xca, 0x6d, 0x72, - 0x34, 0x17, 0xde, 0xce, 0x2b, 0x9e, 0x43, 0xb0, 0x4a, 0xf3, 0x3d, 0xc7, 0x55, 0x9a, 0x13, 0x07, - 0x0f, 0xf5, 0xf1, 0x30, 0xc6, 0xb3, 0x31, 0xdf, 0x81, 0x77, 0x9e, 0x29, 0x47, 0x8f, 0x75, 0xe8, - 0xf1, 0x0f, 0x20, 0x30, 0x55, 0xd9, 0xf3, 0xde, 0x63, 0x7e, 0x0f, 0x46, 0x4f, 0xb3, 0x8d, 0xa8, - 0x55, 0xba, 0x29, 0xe9, 0x29, 0x3c, 0x74, 0x86, 0xf8, 0x7b, 0x98, 0x98, 0x9b, 0xba, 0x5a, 0x57, - 0x42, 0xdd, 0x78, 0xd9, 0xff, 0x56, 0xe5, 0x9b, 0x2f, 0xfd, 0x2b, 0x03, 0x5f, 0xfb, 0xac, 0x8b, - 0xed, 0x5d, 0xba, 0xb0, 0x4f, 0x77, 0xa5, 0x68, 0x99, 0xd2, 0x99, 0x47, 0x30, 0xbe, 0x52, 0x55, - 0x56, 0xbc, 0x5c, 0xa5, 0x79, 0x23, 0xda, 0x44, 0x5d, 0x93, 0xd6, 0xb8, 0x2c, 0x94, 0x71, 0xfb, - 0x24, 0x63, 0x8f, 0xb5, 0xc6, 0x73, 0x29, 0x73, 0xe3, 0xec, 0x47, 0x2c, 0x09, 0xd0, 0x19, 0xf8, - 0x0c, 0xe0, 0x71, 0x2e, 0xd3, 0x36, 0x76, 0x10, 0xb1, 0x84, 0x61, 0xc7, 0x12, 0xdf, 0x87, 0xa1, - 0x66, 0xfa, 0x75, 0x5a, 0x3a, 0xb5, 0xec, 0x16, 0xb5, 0xf1, 0x1b, 0x06, 0x27, 0xdf, 0x36, 0xa2, - 0xda, 0xa1, 0xf8, 0xb1, 0x11, 0x35, 0x55, 0x85, 0xb0, 0x6d, 0x25, 0x02, 0xfc, 0x2e, 0x0c, 0xae, - 0x5e, 0xa5, 0xd5, 0xda, 0xbc, 0x9d, 0x8f, 0x2d, 0xd2, 0x5a, 0xdd, 0x9b, 0xd7, 0xa4, 0x35, 0xc0, - 0xae, 0x49, 0x47, 0xa2, 0xd8, 0x48, 0x65, 0xc5, 0xb4, 0x88, 0x27, 0xf0, 0xee, 0xc5, 0xeb, 0xe7, - 0x79, 0xb3, 0x16, 0x28, 0xb7, 0x26, 0x7a, 0x40, 0x17, 0x8e, 0xcd, 0xfc, 0x63, 0x98, 0xb6, 0x26, - 0x3b, 0xbd, 0x43, 0xba, 0x78, 0x64, 0x8d, 0x7f, 0x66, 0x30, 0x69, 0xa5, 0xd4, 0xa5, 0x2c, 0x6a, - 0xa1, 0xeb, 0x75, 0x51, 0x55, 0xb6, 0x5e, 0x17, 0x55, 0xc5, 0xef, 0xc3, 0x10, 0x45, 0xdd, 0xe4, - 0xca, 0x36, 0xc1, 0x7b, 0xee, 0x59, 0x6c, 0x6c, 0x93, 0x2b, 0xb4, 0xb7, 0xf8, 0x67, 0x30, 0x3d, - 0x68, 0x2a, 0x33, 0xfd, 0xe3, 0xf9, 0xfb, 0x2e, 0xee, 0xc0, 0x8f, 0x47, 0xd7, 0xe3, 0x3f, 0x7a, - 0x30, 0xee, 0x64, 0xe6, 0x1f, 0xd2, 0x2e, 0x22, 0x4e, 0xe3, 0xf9, 0xc4, 0x65, 0xd1, 0x93, 0x46, - 0x5b, 0xea, 0x04, 0xd8, 0x65, 0xdb, 0x4f, 0xec, 0x52, 0x57, 0x51, 0x6f, 0x09, 0xfb, 0xd9, 0x4e, - 0x15, 0xb5, 0x19, 0x8d, 0x93, 0x36, 0xdb, 0xab, 0xb4, 0x78, 0x29, 0xd6, 0xd4, 0x4f, 0x01, 0x5a, - 0xc8, 0xcf, 0xdc, 0x7c, 0x52, 0x01, 0x0e, 0x46, 0xdc, 0x7a, 0xd0, 0xcd, 0xb0, 0x6d, 0x68, 0x5d, - 0x8b, 0x49, 0xdb, 0xd0, 0xba, 0x84, 0x7a, 0x36, 0xf5, 0xc3, 0x53, 0xf1, 0x0d, 0xe2, 0x0f, 0x60, - 0xec, 0x36, 0x49, 0x1d, 0x06, 0xc4, 0xf0, 0xd4, 0xa5, 0x77, 0x4e, 0xec, 0x5e, 0xe4, 0x9f, 0x1f, - 0xef, 0xcc, 0x70, 0x44, 0xcc, 0xc2, 0x83, 0xd7, 0xe8, 0xf8, 0xf1, 0xe8, 0x7e, 0xfc, 0x17, 0x83, - 0xc9, 0x72, 0x53, 0xca, 0x4a, 0x75, 0xda, 0x76, 0x59, 0xac, 0xc5, 0x6b, 0xdb, 0xb6, 0x04, 0xdc, - 0x5e, 0xec, 0x1d, 0xed, 0x45, 0x6a, 0x5f, 0x6a, 0x57, 0x1f, 0x0d, 0xe8, 0xa8, 0xf4, 0x0f, 0x54, - 0xde, 0x83, 0x91, 0x5d, 0x40, 0x75, 0xd8, 0x27, 0x97, 0x33, 0xe8, 0x81, 0xdc, 0x6f, 0x20, 0xdd, - 0xc1, 0x5e, 0xe2, 0x61, 0xc7, 0xa2, 0x2b, 0x83, 0x72, 0x4b, 0xcb, 0x7f, 0x48, 0xcb, 0xdf, 0x42, - 0x1d, 0x69, 0xd2, 0x90, 0x33, 0x20, 0x67, 0xc7, 0x12, 0xff, 0xc6, 0x80, 0x1b, 0x8d, 0x34, 0xda, - 0xff, 0x9f, 0xd0, 0xdb, 0x05, 0xdd, 0x85, 0x01, 0x7d, 0xcf, 0x8a, 0x69, 0xd1, 0x11, 0xdd, 0xe1, - 0x0d, 0xba, 0xe7, 0x10, 0xb6, 0x15, 0x91, 0xa9, 0xde, 0x74, 0x2d, 0xdf, 0x55, 0x26, 0xb6, 0xba, - 0xa9, 0x2e, 0xd3, 0x8d, 0x68, 0x29, 0xd3, 0x59, 0xdb, 0x16, 0xa9, 0x4a, 0x89, 0xf0, 0x09, 0xd2, - 0x39, 0x7e, 0x01, 0xa7, 0x6f, 0xcb, 0x41, 0x3f, 0x23, 0xb9, 0x48, 0xcd, 0x24, 0x07, 0x68, 0x00, - 0x7f, 0x08, 0xfd, 0x9f, 0x32, 0xb1, 0xb5, 0x93, 0x1c, 0xbb, 0xee, 0xf9, 0x37, 0x22, 0x68, 0x02, - 0xce, 0xef, 0xbc, 0xb9, 0x9e, 0xb1, 0xdf, 0xaf, 0x67, 0xec, 0xcf, 0xeb, 0x19, 0xfb, 0xe5, 0xef, - 0xd9, 0x3b, 0xcf, 0x06, 0xf4, 0x97, 0xe1, 0x93, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, 0x60, - 0x42, 0x9e, 0x42, 0x08, 0x00, 0x00, +var fileDescriptor_public_f65cfea24ac19f54 = []byte{ + // 880 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x8e, 0x1b, 0x45, + 0x10, 0xa6, 0x3d, 0x63, 0x7b, 0x5c, 0x5e, 0x9b, 0xa8, 0xe5, 0x84, 0x11, 0x8a, 0x8c, 0x35, 0x42, + 0x68, 0xb8, 0x6c, 0x24, 0x23, 0xa1, 0x9c, 0xf8, 0xd9, 0x78, 0x83, 0xac, 0xc0, 0x0a, 0x6a, 0x57, + 0x46, 0x1c, 0x3b, 0x71, 0x27, 0x19, 0x69, 0x3c, 0x6d, 0x66, 0x7a, 0x70, 0xf6, 0x39, 0xb8, 0xf0, + 0x08, 0x1c, 0x78, 0x90, 0x1c, 0x11, 0x4f, 0x00, 0xcb, 0x8b, 0xa0, 0xae, 0x9e, 0xde, 0x1e, 0x7b, + 0x97, 0x08, 0xa1, 0xdc, 0xea, 0xab, 0xea, 0xaa, 0xa9, 0xaf, 0xfe, 0x6c, 0x38, 0xda, 0xd6, 0x4f, + 0xf3, 0xec, 0xd9, 0xf1, 0xb6, 0x54, 0x5a, 0xf1, 0x28, 0x2b, 0xb4, 0x2c, 0x0b, 0x91, 0x27, 0x3f, + 0x40, 0x80, 0x6a, 0xc7, 0x63, 0xe8, 0x3f, 0x52, 0x79, 0xbd, 0x29, 0xaa, 0x98, 0xcd, 0x82, 0x34, + 0x44, 0x07, 0xf9, 0x87, 0xd0, 0xfd, 0x52, 0xeb, 0xb2, 0x8a, 0x3b, 0xb3, 0x20, 0x1d, 0xce, 0xc7, + 0xc7, 0xce, 0xf5, 0xd8, 0xa8, 0xd1, 0x1a, 0x39, 0x87, 0xf0, 0x89, 0xbc, 0xac, 0xe2, 0x60, 0x16, + 0xa4, 0x03, 0x24, 0x39, 0x79, 0x08, 0x63, 0x54, 0xbb, 0xe5, 0x5a, 0x16, 0x3a, 0x7b, 0x9e, 0x49, + 0xfb, 0x0a, 0xd5, 0xce, 0x7d, 0x82, 0xe4, 0x6b, 0xcf, 0x4e, 0xcb, 0xf3, 0x33, 0x08, 0xbf, 0x15, + 0x59, 0xc9, 0xc7, 0xd0, 0x59, 0x2e, 0x62, 0x36, 0x63, 0x69, 0x88, 0x9d, 0xe5, 0x82, 0x4f, 0xa0, + 0xfb, 0x48, 0xd5, 0x85, 0x8e, 0x3b, 0xa4, 0xb2, 0x80, 0xdf, 0x81, 0xe0, 0x89, 0xbc, 0x8c, 0x83, + 0x19, 0x4b, 0x07, 0x68, 0xc4, 0xe4, 0x0c, 0xa2, 0xc7, 0x99, 0xcc, 0xd7, 0x86, 0xd9, 0x04, 0xba, + 0x24, 0x53, 0x98, 0x01, 0x5a, 0x60, 0xb4, 0x26, 0xb7, 0x85, 0x8b, 0x44, 0x80, 0xdf, 0x83, 0x1e, + 0xaa, 0x9d, 0x0f, 0xd6, 0xa0, 0xe4, 0x6b, 0x80, 0xaf, 0x4a, 0x55, 0x6f, 0xed, 0xf7, 0x52, 0xe8, + 0x12, 0x22, 0x1a, 0xc3, 0x39, 0xf7, 0x15, 0x71, 0x1f, 0x45, 0xfb, 0xe0, 0xf6, 0x7c, 0x93, 0x39, + 0x44, 0x2b, 0x91, 0x5f, 0xe7, 0xbe, 0x12, 0x39, 0xe5, 0x16, 0xa0, 0x11, 0xf7, 0x7d, 0x02, 0xe7, + 0xf3, 0x3d, 0x8c, 0x6c, 0x43, 0x4c, 0xb9, 0xcf, 0xa5, 0xbe, 0x51, 0x9a, 0xff, 0xd6, 0xa6, 0x9b, + 0xa5, 0xfa, 0x95, 0x41, 0x68, 0x6c, 0xce, 0xc4, 0xae, 0x4d, 0xa6, 0x33, 0x17, 0x97, 0x5b, 0xd9, + 0x24, 0x4f, 0x32, 0x9f, 0xc1, 0xf0, 0x5c, 0x97, 0x59, 0xf1, 0x62, 0x25, 0xf2, 0x5a, 0x36, 0x81, + 0xda, 0x2a, 0xfe, 0x3e, 0x44, 0xcb, 0x42, 0x5b, 0x73, 0x48, 0x14, 0xae, 0x31, 0xbf, 0x0f, 0x83, + 0x13, 0xa5, 0x72, 0x6b, 0xec, 0xce, 0x58, 0x1a, 0xa1, 0x57, 0xf0, 0x29, 0xc0, 0xe3, 0x5c, 0x89, + 0xc6, 0xb7, 0x37, 0x63, 0x29, 0xc3, 0x96, 0x26, 0x79, 0x00, 0x7d, 0x93, 0xe9, 0x37, 0x62, 0xeb, + 0xd9, 0xb2, 0x37, 0xb0, 0x4d, 0x5e, 0x33, 0x38, 0xfa, 0xae, 0x96, 0xe5, 0x25, 0xca, 0x1f, 0x6b, + 0x59, 0x69, 0x53, 0x5b, 0xc2, 0x6e, 0x16, 0x08, 0x98, 0xae, 0x9f, 0xbf, 0x14, 0xe5, 0xda, 0xd6, + 0x2e, 0xc4, 0x06, 0x19, 0xae, 0xbe, 0xe6, 0x15, 0x71, 0x8d, 0xb0, 0xad, 0xa2, 0x79, 0x91, 0x1b, + 0xa5, 0x1d, 0x99, 0x06, 0xf1, 0x14, 0xde, 0x3d, 0x7d, 0xf5, 0x2c, 0xaf, 0xd7, 0x12, 0xd5, 0xce, + 0x7a, 0xf7, 0xe8, 0xc1, 0xa1, 0x9a, 0x7f, 0x04, 0xe3, 0x46, 0xe5, 0xd6, 0xaf, 0x4f, 0x0f, 0x0f, + 0xb4, 0xc9, 0xcf, 0x0c, 0x46, 0x0d, 0x95, 0x6a, 0xab, 0x8a, 0x4a, 0x9a, 0x7e, 0x9d, 0x96, 0xa5, + 0xeb, 0xd7, 0x69, 0x59, 0xf2, 0x07, 0xd0, 0x47, 0x59, 0xd5, 0xb9, 0x76, 0x43, 0x70, 0xd7, 0x97, + 0xc5, 0xf9, 0xd6, 0xb9, 0x46, 0xf7, 0x8a, 0x7f, 0x0e, 0xe3, 0xbd, 0xa1, 0xb2, 0xeb, 0x3b, 0x9c, + 0xbf, 0xe7, 0xfd, 0xf6, 0xec, 0x78, 0xf0, 0x3c, 0xf9, 0xa3, 0x03, 0xc3, 0x56, 0x64, 0xfe, 0x01, + 0x1d, 0x13, 0xca, 0x69, 0x38, 0x1f, 0xf9, 0x28, 0x66, 0x25, 0xe8, 0xcc, 0x1c, 0x01, 0x3b, 0x6b, + 0xe6, 0x89, 0x9d, 0x99, 0x2e, 0x9a, 0x35, 0x77, 0x9f, 0x6d, 0x75, 0xd1, 0xa8, 0xd1, 0x1a, 0xe9, + 0x34, 0xbd, 0x14, 0xc5, 0x0b, 0xb9, 0xa6, 0x79, 0x8a, 0xd0, 0x41, 0x7e, 0xec, 0x17, 0x89, 0x1a, + 0xb0, 0xb7, 0x8b, 0xce, 0x82, 0x7e, 0xd9, 0xdc, 0x40, 0x9b, 0x5e, 0x8c, 0x9a, 0x81, 0xb6, 0x2b, + 0xbf, 0x5c, 0x98, 0xc2, 0x53, 0xf3, 0x2d, 0xe2, 0x9f, 0xc2, 0xd0, 0xaf, 0x7c, 0x15, 0x47, 0x94, + 0xe1, 0xc4, 0x87, 0xf7, 0x46, 0x6c, 0x3f, 0xe4, 0x5f, 0x1c, 0x1e, 0xbd, 0x78, 0x40, 0x99, 0xc5, + 0x7b, 0xd5, 0x68, 0xd9, 0xf1, 0xe0, 0x7d, 0xf2, 0x17, 0x83, 0xd1, 0x72, 0xb3, 0x55, 0xa5, 0x6e, + 0x8d, 0xed, 0xb2, 0x58, 0xcb, 0x57, 0x6e, 0x6c, 0x09, 0xf8, 0xc3, 0xd6, 0x39, 0x38, 0x6c, 0x34, + 0xbe, 0x34, 0xae, 0x21, 0x5a, 0xd0, 0x62, 0x19, 0xee, 0xb1, 0xbc, 0x0f, 0x03, 0xdb, 0x52, 0x63, + 0xea, 0x92, 0xc9, 0x2b, 0xcc, 0x42, 0x5e, 0x64, 0x1b, 0x59, 0x69, 0xb1, 0xd9, 0x9a, 0x09, 0x0e, + 0xd2, 0x00, 0x5b, 0x1a, 0xd3, 0x19, 0x7b, 0x20, 0x6d, 0xf1, 0x06, 0xe8, 0xa0, 0xf1, 0xb4, 0x61, + 0xc8, 0x18, 0x91, 0xb1, 0xa5, 0x49, 0x7e, 0x63, 0xc0, 0x2d, 0x47, 0x5a, 0xed, 0xb7, 0x47, 0xf4, + 0xcd, 0x84, 0xee, 0x41, 0x8f, 0xbe, 0xe7, 0xc8, 0x34, 0xe8, 0x20, 0xdd, 0xfe, 0x8d, 0x74, 0x57, + 0x30, 0xb9, 0x28, 0x45, 0x51, 0xe5, 0x42, 0x4b, 0xa3, 0xf8, 0x3f, 0xf9, 0xde, 0xf6, 0x0b, 0xf9, + 0x31, 0xdc, 0x3d, 0x88, 0xeb, 0x97, 0xdb, 0x10, 0x08, 0x88, 0x80, 0x11, 0x93, 0x13, 0x88, 0x9b, + 0xa1, 0x50, 0xc2, 0x1c, 0xdb, 0x26, 0x85, 0x55, 0x26, 0x77, 0x26, 0xf4, 0x99, 0xd8, 0xc8, 0x26, + 0x0b, 0x92, 0x8d, 0x6e, 0x21, 0xb4, 0xa0, 0x1c, 0x8e, 0x90, 0xe4, 0xe4, 0x39, 0x4c, 0x6e, 0x8b, + 0x41, 0x3f, 0x39, 0xb9, 0x14, 0xf6, 0x98, 0x44, 0x68, 0x01, 0x7f, 0x08, 0xdd, 0x9f, 0x32, 0xb9, + 0x73, 0xc7, 0x24, 0xf1, 0x03, 0xfc, 0x6f, 0x89, 0xa0, 0x75, 0x38, 0xb9, 0xf3, 0xfa, 0x6a, 0xca, + 0x7e, 0xbf, 0x9a, 0xb2, 0x3f, 0xaf, 0xa6, 0xec, 0x97, 0xbf, 0xa7, 0xef, 0x3c, 0xed, 0xd1, 0xdf, + 0x8e, 0x4f, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0x9f, 0x87, 0xba, 0x9b, 0x86, 0x08, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 67cef2a40..592874b95 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -22,6 +22,7 @@ message Pair { message FieldRow{ string Field = 1; uint64 RowID = 2; + string RowKey = 3; } message GroupCount{ @@ -34,12 +35,6 @@ message ValCount { int64 Count = 2; } -message Bit { - uint64 RowID = 1; - uint64 ColumnID = 2; - int64 Timestamp = 3; -} - message ColumnAttrSet { uint64 ID = 1; string Key = 3; @@ -106,6 +101,16 @@ message ImportValueRequest { repeated int64 Values = 6; } +message TranslateKeysRequest { + string Index = 1; + string Field = 2; + repeated string Keys = 3; +} + +message TranslateKeysResponse { + repeated uint64 IDs = 3; +} + message ImportRoaringRequestView { string Name = 1; bytes Data = 2; diff --git a/pilosa.go b/pilosa.go index d2fdf8144..550696a31 100644 --- a/pilosa.go +++ b/pilosa.go @@ -49,14 +49,16 @@ var ( ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") - ErrReservedName = errors.New("reserved index or field name") - // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") + ErrQueryCancelled = errors.New("query cancelled") + ErrQueryTimeout = errors.New("query timeout") ErrTooManyWrites = errors.New("too many write commands") - ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") + // TODO(2.0) poorly named - used when a *node* doesn't own a shard. Probably + // we won't need this error at all by 2.0 though. + ErrClusterDoesNotOwnShard = errors.New("node does not own shard") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") @@ -129,9 +131,6 @@ const TimeFormat = "2006-01-02T15:04" // validateName ensures that the name is a valid format. func validateName(name string) error { - if name == existenceFieldName { - return ErrReservedName - } if !nameRegexp.Match([]byte(name)) { return ErrName } diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index e1bea3199..40e33ac4b 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -20,7 +20,7 @@ import ( func TestValidateName(t *testing.T) { names := []string{ - "a", "ab", "ab1", "b-c", "d_e", + "a", "ab", "ab1", "b-c", "d_e", "exists", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } for _, name := range names { @@ -33,8 +33,7 @@ func TestValidateName(t *testing.T) { func TestValidateNameInvalid(t *testing.T) { names := []string{ "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - "exists", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists", } for _, name := range names { if validateName(name) == nil { diff --git a/pql/ast.go b/pql/ast.go index 0e2ff6aef..8b23c708b 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -27,28 +27,34 @@ import ( type Query struct { Calls []*Call - lastField string - lastCond Token - inList bool - callStack []*Call - + callStack []*callStackElem conditional []string } func (q *Query) startCall(name string) { newCall := &Call{Name: name} - q.callStack = append(q.callStack, newCall) + q.callStack = append(q.callStack, &callStackElem{call: newCall}) if len(q.callStack) == 1 { q.Calls = append(q.Calls, newCall) - } else { - calls := q.callStack[len(q.callStack)-2].Children - q.callStack[len(q.callStack)-2].Children = append(calls, newCall) + } else if prevElem := q.callStack[len(q.callStack)-2]; prevElem.lastField == "" { + prevElem.call.Children = append(prevElem.call.Children, newCall) } } -func (q *Query) endCall() { +// endCall removes the last element from the call stack and returns the call. +func (q *Query) endCall() *Call { + elem := q.callStack[len(q.callStack)-1] + q.callStack[len(q.callStack)-1] = nil q.callStack = q.callStack[:len(q.callStack)-1] + return elem.call +} + +func (q *Query) lastCallStackElem() *callStackElem { + if len(q.callStack) == 0 { + return nil + } + return q.callStack[len(q.callStack)-1] } func (q *Query) addPosNum(key, value string) { @@ -63,9 +69,9 @@ func (q *Query) addPosStr(key, value string) { func (q *Query) startConditional() { q.conditional = make([]string, 0) - call := q.callStack[len(q.callStack)-1] - if call.Args == nil { - call.Args = make(map[string]interface{}) + elem := q.lastCallStackElem() + if elem.call.Args == nil { + elem.call.Args = make(map[string]interface{}) } } @@ -85,51 +91,52 @@ func (q *Query) endConditional() { if q.conditional[1] == "<" { low++ } - if q.conditional[3] == "<=" { - high++ + if q.conditional[3] == "<" { + high-- } - call := q.callStack[len(q.callStack)-1] - call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} + elem := q.lastCallStackElem() + elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} q.conditional = nil } func (q *Query) addField(field string) { - if q.lastField != "" { - panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) + elem := q.lastCallStackElem() + if elem == nil || elem.lastField != "" { + panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField)) } - q.lastField = field - call := q.callStack[len(q.callStack)-1] - if call.Args == nil { - call.Args = make(map[string]interface{}) + elem.lastField = field + if elem.call.Args == nil { + elem.call.Args = make(map[string]interface{}) } } func (q *Query) addVal(val interface{}) { - if q.lastField == "" { + elem := q.lastCallStackElem() + if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) } - call := q.callStack[len(q.callStack)-1] - if q.inList { - list := call.Args[q.lastField].([]interface{}) - call.Args[q.lastField] = append(list, val) + if elem.inList { + list := elem.call.Args[elem.lastField].([]interface{}) + elem.call.Args[elem.lastField] = append(list, val) return } - if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: val, } } else { - call.Args[q.lastField] = val + elem.call.Args[elem.lastField] = val } - q.lastField = "" - q.lastCond = ILLEGAL + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) addNumVal(val string) { - if q.lastField == "" { + elem := q.lastCallStackElem() + if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val)) } var ival interface{} @@ -142,70 +149,70 @@ func (q *Query) addNumVal(val string) { if err != nil { panic(err) } - call := q.callStack[len(q.callStack)-1] - if q.inList { - if q.lastCond != ILLEGAL { - list := call.Args[q.lastField].(*Condition).Value.([]interface{}) - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + if elem.inList { + if elem.lastCond != ILLEGAL { + list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{}) + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: append(list, ival), } } else { - list := call.Args[q.lastField].([]interface{}) - call.Args[q.lastField] = append(list, ival) + list := elem.call.Args[elem.lastField].([]interface{}) + elem.call.Args[elem.lastField] = append(list, ival) } return - } else if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + } else if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: ival, } } else { - call.Args[q.lastField] = ival + elem.call.Args[elem.lastField] = ival } - q.lastField = "" - q.lastCond = ILLEGAL + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) startList() { - call := q.callStack[len(q.callStack)-1] - if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + elem := q.lastCallStackElem() + if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: make([]interface{}, 0), } } else { - call.Args[q.lastField] = make([]interface{}, 0) + elem.call.Args[elem.lastField] = make([]interface{}, 0) } - q.inList = true + elem.inList = true } func (q *Query) endList() { - q.inList = false - q.lastField = "" - q.lastCond = ILLEGAL + elem := q.lastCallStackElem() + elem.inList = false + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) addGT() { - q.lastCond = GT + q.lastCallStackElem().lastCond = GT } func (q *Query) addLT() { - q.lastCond = LT + q.lastCallStackElem().lastCond = LT } func (q *Query) addGTE() { - q.lastCond = GTE + q.lastCallStackElem().lastCond = GTE } func (q *Query) addLTE() { - q.lastCond = LTE + q.lastCallStackElem().lastCond = LTE } func (q *Query) addEQ() { - q.lastCond = EQ + q.lastCallStackElem().lastCond = EQ } func (q *Query) addNEQ() { - q.lastCond = NEQ + q.lastCallStackElem().lastCond = NEQ } func (q *Query) addBTWN() { - q.lastCond = BETWEEN + q.lastCallStackElem().lastCond = BETWEEN } // WriteCallN returns the number of mutating calls. @@ -229,6 +236,13 @@ func (q *Query) String() string { return strings.Join(a, "\n") } +type callStackElem struct { + call *Call + lastField string + lastCond Token + inList bool +} + // Call represents a function call in the AST. type Call struct { Name string @@ -241,13 +255,25 @@ type Call struct { // Returns the field as a string if present, or an error if not. func (c *Call) FieldArg() (string, error) { for arg := range c.Args { - if !strings.HasPrefix(arg, "_") { + if !IsReservedArg(arg) { return arg, nil } } return "", fmt.Errorf("No field argument specified") } +func IsReservedArg(name string) bool { + if strings.HasPrefix(name, "_") { + return true + } + switch name { + case "from", "to": + return true + default: + return false + } +} + // BoolArg is for reading the value at key from call.Args as a bool. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a bool. An error is @@ -277,6 +303,9 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } switch tval := val.(type) { case int64: + if tval < 0 { + return 0, true, fmt.Errorf("value for '%s' must be positive, but got %v", key, tval) + } return uint64(tval), true, nil case uint64: return tval, true, nil @@ -329,6 +358,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// CallArg is for reading the value at key from call.Args as a Call. If the +// key is not in Call.Args, the value of the returned value will be nil, and +// the error will be nil. An error is returned if the value is not a Call. +func (c *Call) CallArg(key string) (*Call, bool, error) { + val, ok := c.Args[key] + if !ok { + return nil, false, nil + } + switch tval := val.(type) { + case *Call: + return tval, true, nil + default: + return nil, true, fmt.Errorf("could not convert %v of type %T to Call in Call.CallArg", tval, tval) + } +} + // keys returns a list of argument keys in sorted order. func (c *Call) keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/pql/pql.peg b/pql/pql.peg index 15d33ac65..66c1d3265 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -13,12 +13,12 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} - / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp arg <- ( field sp '=' sp value / field sp COND sp value + / conditional ) COND <- ( '><' { p.addBTWN() } / '<=' { p.addLTE() } @@ -28,13 +28,12 @@ COND <- ( '><' { p.addBTWN() } / '<' { p.addLT() } / '>' { p.addGT() } ) + conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])} condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} condfield <- sp {p.condAdd(buffer[begin:end])} -timerange <- field sp '=' sp value comma {p.addPosStr("_start", buffer[begin:end])} comma {p.addPosStr("_end", buffer[begin:end])} - value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) @@ -42,8 +41,10 @@ list <- item (comma list)? item <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'true' &(comma / sp close) { p.addVal(true) } / 'false' &(comma / sp close) { p.addVal(false) } + / timestampfmt { p.addVal(buffer[begin:end]) } / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } + / < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } / < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) } / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } @@ -76,5 +77,5 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] -timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt +timestampfmt <- '"' '"' / '\'' '\'' / timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 5c3fb9ee9..8df2fcb72 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -26,7 +26,6 @@ const ( rulecondint rulecondLT rulecondfield - ruletimerange rulevalue rulelist ruleitem @@ -63,9 +62,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 + rulePegText ruleAction14 ruleAction15 - rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -100,7 +99,6 @@ const ( ruleAction47 ruleAction48 ruleAction49 - ruleAction50 ) var rul3s = [...]string{ @@ -115,7 +113,6 @@ var rul3s = [...]string{ "condint", "condLT", "condfield", - "timerange", "value", "list", "item", @@ -152,9 +149,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", + "PegText", "Action14", "Action15", - "PegText", "Action16", "Action17", "Action18", @@ -189,7 +186,6 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", - "Action50", } type token32 struct { @@ -306,7 +302,7 @@ type PQL struct { Buffer string buffer []rune - rules [86]func() bool + rules [84]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -427,79 +423,77 @@ func (p *PQL) Execute() { case ruleAction13: p.endCall() case ruleAction14: - p.startCall("Range") + p.startCall(buffer[begin:end]) case ruleAction15: p.endCall() case ruleAction16: - p.startCall(buffer[begin:end]) - case ruleAction17: - p.endCall() - case ruleAction18: p.addBTWN() - case ruleAction19: + case ruleAction17: p.addLTE() - case ruleAction20: + case ruleAction18: p.addGTE() - case ruleAction21: + case ruleAction19: p.addEQ() - case ruleAction22: + case ruleAction20: p.addNEQ() - case ruleAction23: + case ruleAction21: p.addLT() - case ruleAction24: + case ruleAction22: p.addGT() - case ruleAction25: + case ruleAction23: p.startConditional() - case ruleAction26: + case ruleAction24: p.endConditional() + case ruleAction25: + p.condAdd(buffer[begin:end]) + case ruleAction26: + p.condAdd(buffer[begin:end]) case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.condAdd(buffer[begin:end]) - case ruleAction29: - p.condAdd(buffer[begin:end]) - case ruleAction30: - p.addPosStr("_start", buffer[begin:end]) - case ruleAction31: - p.addPosStr("_end", buffer[begin:end]) - case ruleAction32: p.startList() - case ruleAction33: + case ruleAction29: p.endList() - case ruleAction34: + case ruleAction30: p.addVal(nil) - case ruleAction35: + case ruleAction31: p.addVal(true) - case ruleAction36: + case ruleAction32: p.addVal(false) - case ruleAction37: - p.addNumVal(buffer[begin:end]) - case ruleAction38: - p.addNumVal(buffer[begin:end]) - case ruleAction39: + case ruleAction33: p.addVal(buffer[begin:end]) - case ruleAction40: + case ruleAction34: + p.addNumVal(buffer[begin:end]) + case ruleAction35: + p.addNumVal(buffer[begin:end]) + case ruleAction36: + p.startCall(buffer[begin:end]) + case ruleAction37: + p.addVal(p.endCall()) + case ruleAction38: + p.addVal(buffer[begin:end]) + case ruleAction39: s, _ := strconv.Unquote(buffer[begin:end]) p.addVal(s) - case ruleAction41: + case ruleAction40: p.addVal(buffer[begin:end]) - case ruleAction42: + case ruleAction41: p.addField(buffer[begin:end]) - case ruleAction43: + case ruleAction42: p.addPosStr("_field", buffer[begin:end]) - case ruleAction44: + case ruleAction43: p.addPosNum("_col", buffer[begin:end]) + case ruleAction44: + p.addPosStr("_col", buffer[begin:end]) case ruleAction45: p.addPosStr("_col", buffer[begin:end]) case ruleAction46: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction47: p.addPosNum("_row", buffer[begin:end]) + case ruleAction47: + p.addPosStr("_row", buffer[begin:end]) case ruleAction48: p.addPosStr("_row", buffer[begin:end]) case ruleAction49: - p.addPosStr("_row", buffer[begin:end]) - case ruleAction50: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -612,7 +606,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'a' 'n' 'g' 'e' Action14 open (timerange / conditional / arg) close Action15) / ( Action16 open allargs comma? close Action17))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ( Action14 open allargs comma? close Action15))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -661,7 +655,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction50, position) + add(ruleAction49, position) } add(ruletimestamp, position12) } @@ -747,7 +741,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction47, position) + add(ruleAction46, position) } goto l19 l20: @@ -768,7 +762,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction47, position) } goto l19 l23: @@ -789,7 +783,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction49, position) + add(ruleAction48, position) } } l19: @@ -1075,194 +1069,16 @@ func (p *PQL) Init() { goto l7 l41: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l46 + { + position46 := position + if !_rules[ruleIDENT]() { + goto l5 + } + add(rulePegText, position46) } - position++ - if buffer[position] != rune('a') { - goto l46 - } - position++ - if buffer[position] != rune('n') { - goto l46 - } - position++ - if buffer[position] != rune('g') { - goto l46 - } - position++ - if buffer[position] != rune('e') { - goto l46 - } - position++ { add(ruleAction14, position) } - if !_rules[ruleopen]() { - goto l46 - } - { - position48, tokenIndex48 := position, tokenIndex - { - position50 := position - if !_rules[rulefield]() { - goto l49 - } - if !_rules[rulesp]() { - goto l49 - } - if buffer[position] != rune('=') { - goto l49 - } - position++ - if !_rules[rulesp]() { - goto l49 - } - if !_rules[rulevalue]() { - goto l49 - } - if !_rules[rulecomma]() { - goto l49 - } - { - position51 := position - if !_rules[ruletimestampfmt]() { - goto l49 - } - add(rulePegText, position51) - } - { - add(ruleAction30, position) - } - if !_rules[rulecomma]() { - goto l49 - } - { - position53 := position - if !_rules[ruletimestampfmt]() { - goto l49 - } - add(rulePegText, position53) - } - { - add(ruleAction31, position) - } - add(ruletimerange, position50) - } - goto l48 - l49: - position, tokenIndex = position48, tokenIndex48 - { - position56 := position - { - add(ruleAction25, position) - } - if !_rules[rulecondint]() { - goto l55 - } - if !_rules[rulecondLT]() { - goto l55 - } - { - position58 := position - { - position59 := position - if !_rules[rulefieldExpr]() { - goto l55 - } - add(rulePegText, position59) - } - if !_rules[rulesp]() { - goto l55 - } - { - add(ruleAction29, position) - } - add(rulecondfield, position58) - } - if !_rules[rulecondLT]() { - goto l55 - } - if !_rules[rulecondint]() { - goto l55 - } - { - add(ruleAction26, position) - } - add(ruleconditional, position56) - } - goto l48 - l55: - position, tokenIndex = position48, tokenIndex48 - if !_rules[rulearg]() { - goto l46 - } - } - l48: - if !_rules[ruleclose]() { - goto l46 - } - { - add(ruleAction15, position) - } - goto l7 - l46: - position, tokenIndex = position7, tokenIndex7 - { - position63 := position - { - position64 := position - { - position65, tokenIndex65 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l66 - } - position++ - goto l65 - l66: - position, tokenIndex = position65, tokenIndex65 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l5 - } - position++ - } - l65: - l67: - { - position68, tokenIndex68 := position, tokenIndex - { - position69, tokenIndex69 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l70 - } - position++ - goto l69 - l70: - position, tokenIndex = position69, tokenIndex69 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l71 - } - position++ - goto l69 - l71: - position, tokenIndex = position69, tokenIndex69 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l68 - } - position++ - } - l69: - goto l67 - l68: - position, tokenIndex = position68, tokenIndex68 - } - add(ruleIDENT, position64) - } - add(rulePegText, position63) - } - { - add(ruleAction16, position) - } if !_rules[ruleopen]() { goto l5 } @@ -1270,20 +1086,20 @@ func (p *PQL) Init() { goto l5 } { - position73, tokenIndex73 := position, tokenIndex + position48, tokenIndex48 := position, tokenIndex if !_rules[rulecomma]() { - goto l73 + goto l48 } - goto l74 - l73: - position, tokenIndex = position73, tokenIndex73 + goto l49 + l48: + position, tokenIndex = position48, tokenIndex48 } - l74: + l49: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction17, position) + add(ruleAction15, position) } } l7: @@ -1296,1651 +1112,1799 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position76, tokenIndex76 := position, tokenIndex + position51, tokenIndex51 := position, tokenIndex { - position77 := position + position52 := position { - position78, tokenIndex78 := position, tokenIndex + position53, tokenIndex53 := position, tokenIndex if !_rules[ruleCall]() { - goto l79 + goto l54 } - l80: + l55: { - position81, tokenIndex81 := position, tokenIndex + position56, tokenIndex56 := position, tokenIndex if !_rules[rulecomma]() { - goto l81 + goto l56 } if !_rules[ruleCall]() { - goto l81 + goto l56 } - goto l80 - l81: - position, tokenIndex = position81, tokenIndex81 + goto l55 + l56: + position, tokenIndex = position56, tokenIndex56 } { - position82, tokenIndex82 := position, tokenIndex + position57, tokenIndex57 := position, tokenIndex if !_rules[rulecomma]() { - goto l82 + goto l57 } if !_rules[ruleargs]() { - goto l82 + goto l57 } - goto l83 - l82: - position, tokenIndex = position82, tokenIndex82 + goto l58 + l57: + position, tokenIndex = position57, tokenIndex57 } - l83: - goto l78 - l79: - position, tokenIndex = position78, tokenIndex78 + l58: + goto l53 + l54: + position, tokenIndex = position53, tokenIndex53 if !_rules[ruleargs]() { - goto l84 + goto l59 } - goto l78 - l84: - position, tokenIndex = position78, tokenIndex78 + goto l53 + l59: + position, tokenIndex = position53, tokenIndex53 if !_rules[rulesp]() { - goto l76 + goto l51 } } - l78: - add(ruleallargs, position77) + l53: + add(ruleallargs, position52) } return true - l76: - position, tokenIndex = position76, tokenIndex76 + l51: + position, tokenIndex = position51, tokenIndex51 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position85, tokenIndex85 := position, tokenIndex + position60, tokenIndex60 := position, tokenIndex { - position86 := position + position61 := position if !_rules[rulearg]() { - goto l85 + goto l60 } { - position87, tokenIndex87 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex if !_rules[rulecomma]() { - goto l87 + goto l62 } if !_rules[ruleargs]() { - goto l87 + goto l62 } - goto l88 - l87: - position, tokenIndex = position87, tokenIndex87 + goto l63 + l62: + position, tokenIndex = position62, tokenIndex62 } - l88: + l63: if !_rules[rulesp]() { - goto l85 + goto l60 } - add(ruleargs, position86) + add(ruleargs, position61) } return true - l85: - position, tokenIndex = position85, tokenIndex85 + l60: + position, tokenIndex = position60, tokenIndex60 return false }, - /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ + /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value) / conditional)> */ func() bool { - position89, tokenIndex89 := position, tokenIndex + position64, tokenIndex64 := position, tokenIndex { - position90 := position + position65 := position { - position91, tokenIndex91 := position, tokenIndex + position66, tokenIndex66 := position, tokenIndex if !_rules[rulefield]() { - goto l92 + goto l67 } if !_rules[rulesp]() { - goto l92 + goto l67 } if buffer[position] != rune('=') { - goto l92 + goto l67 } position++ if !_rules[rulesp]() { - goto l92 + goto l67 } if !_rules[rulevalue]() { - goto l92 + goto l67 } - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 + goto l66 + l67: + position, tokenIndex = position66, tokenIndex66 if !_rules[rulefield]() { - goto l89 + goto l68 } if !_rules[rulesp]() { - goto l89 + goto l68 } { - position93 := position + position69 := position { - position94, tokenIndex94 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if buffer[position] != rune('>') { - goto l95 + goto l71 } position++ if buffer[position] != rune('<') { - goto l95 + goto l71 + } + position++ + { + add(ruleAction16, position) + } + goto l70 + l71: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('<') { + goto l73 + } + position++ + if buffer[position] != rune('=') { + goto l73 + } + position++ + { + add(ruleAction17, position) + } + goto l70 + l73: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('>') { + goto l75 + } + position++ + if buffer[position] != rune('=') { + goto l75 } position++ { add(ruleAction18, position) } - goto l94 - l95: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('<') { - goto l97 + goto l70 + l75: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('=') { + goto l77 } position++ if buffer[position] != rune('=') { - goto l97 + goto l77 } position++ { add(ruleAction19, position) } - goto l94 - l97: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('>') { - goto l99 + goto l70 + l77: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('!') { + goto l79 } position++ if buffer[position] != rune('=') { - goto l99 + goto l79 } position++ { add(ruleAction20, position) } - goto l94 - l99: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('=') { - goto l101 - } - position++ - if buffer[position] != rune('=') { - goto l101 + goto l70 + l79: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('<') { + goto l81 } position++ { add(ruleAction21, position) } - goto l94 - l101: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('!') { - goto l103 - } - position++ - if buffer[position] != rune('=') { - goto l103 + goto l70 + l81: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('>') { + goto l68 } position++ { add(ruleAction22, position) } - goto l94 - l103: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('<') { - goto l105 - } - position++ - { - add(ruleAction23, position) - } - goto l94 - l105: - position, tokenIndex = position94, tokenIndex94 - if buffer[position] != rune('>') { - goto l89 - } - position++ - { - add(ruleAction24, position) - } } - l94: - add(ruleCOND, position93) + l70: + add(ruleCOND, position69) } if !_rules[rulesp]() { - goto l89 + goto l68 } if !_rules[rulevalue]() { - goto l89 + goto l68 + } + goto l66 + l68: + position, tokenIndex = position66, tokenIndex66 + { + position84 := position + { + add(ruleAction23, position) + } + if !_rules[rulecondint]() { + goto l64 + } + if !_rules[rulecondLT]() { + goto l64 + } + { + position86 := position + { + position87 := position + if !_rules[rulefieldExpr]() { + goto l64 + } + add(rulePegText, position87) + } + if !_rules[rulesp]() { + goto l64 + } + { + add(ruleAction27, position) + } + add(rulecondfield, position86) + } + if !_rules[rulecondLT]() { + goto l64 + } + if !_rules[rulecondint]() { + goto l64 + } + { + add(ruleAction24, position) + } + add(ruleconditional, position84) } } - l91: - add(rulearg, position90) + l66: + add(rulearg, position65) } return true - l89: - position, tokenIndex = position89, tokenIndex89 + l64: + position, tokenIndex = position64, tokenIndex64 return false }, - /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ + /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ nil, - /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ + /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ func() bool { - position110, tokenIndex110 := position, tokenIndex + position92, tokenIndex92 := position, tokenIndex { - position111 := position + position93 := position { - position112 := position + position94 := position { - position113, tokenIndex113 := position, tokenIndex + position95, tokenIndex95 := position, tokenIndex { - position115, tokenIndex115 := position, tokenIndex + position97, tokenIndex97 := position, tokenIndex if buffer[position] != rune('-') { - goto l115 + goto l97 } position++ - goto l116 - l115: - position, tokenIndex = position115, tokenIndex115 + goto l98 + l97: + position, tokenIndex = position97, tokenIndex97 } - l116: + l98: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l114 + goto l96 } position++ - l117: + l99: { - position118, tokenIndex118 := position, tokenIndex + position100, tokenIndex100 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l118 + goto l100 } position++ - goto l117 - l118: - position, tokenIndex = position118, tokenIndex118 + goto l99 + l100: + position, tokenIndex = position100, tokenIndex100 } - goto l113 - l114: - position, tokenIndex = position113, tokenIndex113 + goto l95 + l96: + position, tokenIndex = position95, tokenIndex95 if buffer[position] != rune('0') { - goto l110 + goto l92 } position++ } - l113: - add(rulePegText, position112) + l95: + add(rulePegText, position94) } if !_rules[rulesp]() { - goto l110 + goto l92 } { - add(ruleAction27, position) + add(ruleAction25, position) } - add(rulecondint, position111) + add(rulecondint, position93) } return true - l110: - position, tokenIndex = position110, tokenIndex110 + l92: + position, tokenIndex = position92, tokenIndex92 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ func() bool { - position120, tokenIndex120 := position, tokenIndex + position102, tokenIndex102 := position, tokenIndex { - position121 := position + position103 := position { - position122 := position + position104 := position { - position123, tokenIndex123 := position, tokenIndex + position105, tokenIndex105 := position, tokenIndex if buffer[position] != rune('<') { - goto l124 + goto l106 } position++ if buffer[position] != rune('=') { - goto l124 + goto l106 } position++ - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 + goto l105 + l106: + position, tokenIndex = position105, tokenIndex105 if buffer[position] != rune('<') { - goto l120 + goto l102 } position++ } - l123: - add(rulePegText, position122) + l105: + add(rulePegText, position104) } if !_rules[rulesp]() { - goto l120 + goto l102 } { - add(ruleAction28, position) + add(ruleAction26, position) } - add(rulecondLT, position121) + add(rulecondLT, position103) } return true - l120: - position, tokenIndex = position120, tokenIndex120 + l102: + position, tokenIndex = position102, tokenIndex102 return false }, - /* 9 condfield <- <( sp Action29)> */ + /* 9 condfield <- <( sp Action27)> */ nil, - /* 10 timerange <- <(field sp '=' sp value comma Action30 comma Action31)> */ - nil, - /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ + /* 10 value <- <(item / (lbrack Action28 list rbrack Action29))> */ func() bool { - position128, tokenIndex128 := position, tokenIndex + position109, tokenIndex109 := position, tokenIndex { - position129 := position + position110 := position { - position130, tokenIndex130 := position, tokenIndex + position111, tokenIndex111 := position, tokenIndex if !_rules[ruleitem]() { - goto l131 + goto l112 } - goto l130 - l131: - position, tokenIndex = position130, tokenIndex130 + goto l111 + l112: + position, tokenIndex = position111, tokenIndex111 { - position132 := position + position113 := position if buffer[position] != rune('[') { - goto l128 + goto l109 } position++ if !_rules[rulesp]() { - goto l128 + goto l109 } - add(rulelbrack, position132) + add(rulelbrack, position113) + } + { + add(ruleAction28, position) + } + if !_rules[rulelist]() { + goto l109 + } + { + position115 := position + if !_rules[rulesp]() { + goto l109 + } + if buffer[position] != rune(']') { + goto l109 + } + position++ + if !_rules[rulesp]() { + goto l109 + } + add(rulerbrack, position115) + } + { + add(ruleAction29, position) + } + } + l111: + add(rulevalue, position110) + } + return true + l109: + position, tokenIndex = position109, tokenIndex109 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position117, tokenIndex117 := position, tokenIndex + { + position118 := position + if !_rules[ruleitem]() { + goto l117 + } + { + position119, tokenIndex119 := position, tokenIndex + if !_rules[rulecomma]() { + goto l119 + } + if !_rules[rulelist]() { + goto l119 + } + goto l120 + l119: + position, tokenIndex = position119, tokenIndex119 + } + l120: + add(rulelist, position118) + } + return true + l117: + position, tokenIndex = position117, tokenIndex117 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (timestampfmt Action33) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action34) / (<('-'? '.' [0-9]+)> Action35) / ( Action36 open allargs comma? close Action37) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action38) / (<('"' doublequotedstring '"')> Action39) / ('\'' '\'' Action40))> */ + func() bool { + position121, tokenIndex121 := position, tokenIndex + { + position122 := position + { + position123, tokenIndex123 := position, tokenIndex + if buffer[position] != rune('n') { + goto l124 + } + position++ + if buffer[position] != rune('u') { + goto l124 + } + position++ + if buffer[position] != rune('l') { + goto l124 + } + position++ + if buffer[position] != rune('l') { + goto l124 + } + position++ + { + position125, tokenIndex125 := position, tokenIndex + { + position126, tokenIndex126 := position, tokenIndex + if !_rules[rulecomma]() { + goto l127 + } + goto l126 + l127: + position, tokenIndex = position126, tokenIndex126 + if !_rules[rulesp]() { + goto l124 + } + if !_rules[ruleclose]() { + goto l124 + } + } + l126: + position, tokenIndex = position125, tokenIndex125 + } + { + add(ruleAction30, position) + } + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('t') { + goto l129 + } + position++ + if buffer[position] != rune('r') { + goto l129 + } + position++ + if buffer[position] != rune('u') { + goto l129 + } + position++ + if buffer[position] != rune('e') { + goto l129 + } + position++ + { + position130, tokenIndex130 := position, tokenIndex + { + position131, tokenIndex131 := position, tokenIndex + if !_rules[rulecomma]() { + goto l132 + } + goto l131 + l132: + position, tokenIndex = position131, tokenIndex131 + if !_rules[rulesp]() { + goto l129 + } + if !_rules[ruleclose]() { + goto l129 + } + } + l131: + position, tokenIndex = position130, tokenIndex130 + } + { + add(ruleAction31, position) + } + goto l123 + l129: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('f') { + goto l134 + } + position++ + if buffer[position] != rune('a') { + goto l134 + } + position++ + if buffer[position] != rune('l') { + goto l134 + } + position++ + if buffer[position] != rune('s') { + goto l134 + } + position++ + if buffer[position] != rune('e') { + goto l134 + } + position++ + { + position135, tokenIndex135 := position, tokenIndex + { + position136, tokenIndex136 := position, tokenIndex + if !_rules[rulecomma]() { + goto l137 + } + goto l136 + l137: + position, tokenIndex = position136, tokenIndex136 + if !_rules[rulesp]() { + goto l134 + } + if !_rules[ruleclose]() { + goto l134 + } + } + l136: + position, tokenIndex = position135, tokenIndex135 } { add(ruleAction32, position) } - if !_rules[rulelist]() { - goto l128 - } - { - position134 := position - if !_rules[rulesp]() { - goto l128 - } - if buffer[position] != rune(']') { - goto l128 - } - position++ - if !_rules[rulesp]() { - goto l128 - } - add(rulerbrack, position134) + goto l123 + l134: + position, tokenIndex = position123, tokenIndex123 + if !_rules[ruletimestampfmt]() { + goto l139 } { add(ruleAction33, position) } - } - l130: - add(rulevalue, position129) - } - return true - l128: - position, tokenIndex = position128, tokenIndex128 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position136, tokenIndex136 := position, tokenIndex - { - position137 := position - if !_rules[ruleitem]() { - goto l136 - } - { - position138, tokenIndex138 := position, tokenIndex - if !_rules[rulecomma]() { - goto l138 - } - if !_rules[rulelist]() { - goto l138 - } - goto l139 - l138: - position, tokenIndex = position138, tokenIndex138 - } - l139: - add(rulelist, position137) - } - return true - l136: - position, tokenIndex = position136, tokenIndex136 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action39) / (<('"' doublequotedstring '"')> Action40) / ('\'' '\'' Action41))> */ - func() bool { - position140, tokenIndex140 := position, tokenIndex - { - position141 := position - { - position142, tokenIndex142 := position, tokenIndex - if buffer[position] != rune('n') { - goto l143 - } - position++ - if buffer[position] != rune('u') { - goto l143 - } - position++ - if buffer[position] != rune('l') { - goto l143 - } - position++ - if buffer[position] != rune('l') { - goto l143 - } - position++ + goto l123 + l139: + position, tokenIndex = position123, tokenIndex123 { - position144, tokenIndex144 := position, tokenIndex + position142 := position { - position145, tokenIndex145 := position, tokenIndex - if !_rules[rulecomma]() { + position143, tokenIndex143 := position, tokenIndex + if buffer[position] != rune('-') { + goto l143 + } + position++ + goto l144 + l143: + position, tokenIndex = position143, tokenIndex143 + } + l144: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l141 + } + position++ + l145: + { + position146, tokenIndex146 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l146 } + position++ goto l145 l146: - position, tokenIndex = position145, tokenIndex145 - if !_rules[rulesp]() { - goto l143 - } - if !_rules[ruleclose]() { - goto l143 - } + position, tokenIndex = position146, tokenIndex146 } - l145: - position, tokenIndex = position144, tokenIndex144 + { + position147, tokenIndex147 := position, tokenIndex + if buffer[position] != rune('.') { + goto l147 + } + position++ + l149: + { + position150, tokenIndex150 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l150 + } + position++ + goto l149 + l150: + position, tokenIndex = position150, tokenIndex150 + } + goto l148 + l147: + position, tokenIndex = position147, tokenIndex147 + } + l148: + add(rulePegText, position142) } { add(ruleAction34, position) } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 - if buffer[position] != rune('t') { - goto l148 - } - position++ - if buffer[position] != rune('r') { - goto l148 - } - position++ - if buffer[position] != rune('u') { - goto l148 - } - position++ - if buffer[position] != rune('e') { - goto l148 - } - position++ + goto l123 + l141: + position, tokenIndex = position123, tokenIndex123 { - position149, tokenIndex149 := position, tokenIndex + position153 := position { - position150, tokenIndex150 := position, tokenIndex - if !_rules[rulecomma]() { - goto l151 - } - goto l150 - l151: - position, tokenIndex = position150, tokenIndex150 - if !_rules[rulesp]() { - goto l148 - } - if !_rules[ruleclose]() { - goto l148 + position154, tokenIndex154 := position, tokenIndex + if buffer[position] != rune('-') { + goto l154 } + position++ + goto l155 + l154: + position, tokenIndex = position154, tokenIndex154 } - l150: - position, tokenIndex = position149, tokenIndex149 + l155: + if buffer[position] != rune('.') { + goto l152 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l152 + } + position++ + l156: + { + position157, tokenIndex157 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l157 + } + position++ + goto l156 + l157: + position, tokenIndex = position157, tokenIndex157 + } + add(rulePegText, position153) } { add(ruleAction35, position) } - goto l142 - l148: - position, tokenIndex = position142, tokenIndex142 - if buffer[position] != rune('f') { - goto l153 - } - position++ - if buffer[position] != rune('a') { - goto l153 - } - position++ - if buffer[position] != rune('l') { - goto l153 - } - position++ - if buffer[position] != rune('s') { - goto l153 - } - position++ - if buffer[position] != rune('e') { - goto l153 - } - position++ + goto l123 + l152: + position, tokenIndex = position123, tokenIndex123 { - position154, tokenIndex154 := position, tokenIndex - { - position155, tokenIndex155 := position, tokenIndex - if !_rules[rulecomma]() { - goto l156 - } - goto l155 - l156: - position, tokenIndex = position155, tokenIndex155 - if !_rules[rulesp]() { - goto l153 - } - if !_rules[ruleclose]() { - goto l153 - } + position160 := position + if !_rules[ruleIDENT]() { + goto l159 } - l155: - position, tokenIndex = position154, tokenIndex154 + add(rulePegText, position160) } { add(ruleAction36, position) } - goto l142 - l153: - position, tokenIndex = position142, tokenIndex142 + if !_rules[ruleopen]() { + goto l159 + } + if !_rules[ruleallargs]() { + goto l159 + } { - position159 := position - { - position160, tokenIndex160 := position, tokenIndex - if buffer[position] != rune('-') { - goto l160 - } - position++ - goto l161 - l160: - position, tokenIndex = position160, tokenIndex160 - } - l161: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l158 - } - position++ - l162: - { - position163, tokenIndex163 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 - } - position++ + position162, tokenIndex162 := position, tokenIndex + if !_rules[rulecomma]() { goto l162 - l163: - position, tokenIndex = position163, tokenIndex163 } - { - position164, tokenIndex164 := position, tokenIndex - if buffer[position] != rune('.') { - goto l164 - } - position++ - l166: - { - position167, tokenIndex167 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l167 - } - position++ - goto l166 - l167: - position, tokenIndex = position167, tokenIndex167 - } - goto l165 - l164: - position, tokenIndex = position164, tokenIndex164 - } - l165: - add(rulePegText, position159) + goto l163 + l162: + position, tokenIndex = position162, tokenIndex162 + } + l163: + if !_rules[ruleclose]() { + goto l159 } { add(ruleAction37, position) } - goto l142 - l158: - position, tokenIndex = position142, tokenIndex142 + goto l123 + l159: + position, tokenIndex = position123, tokenIndex123 { - position170 := position + position166 := position { - position171, tokenIndex171 := position, tokenIndex - if buffer[position] != rune('-') { + position169, tokenIndex169 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l170 + } + position++ + goto l169 + l170: + position, tokenIndex = position169, tokenIndex169 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l171 } position++ - goto l172 + goto l169 l171: - position, tokenIndex = position171, tokenIndex171 - } - l172: - if buffer[position] != rune('.') { - goto l169 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l169 - } - position++ - l173: - { - position174, tokenIndex174 := position, tokenIndex + position, tokenIndex = position169, tokenIndex169 if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l172 + } + position++ + goto l169 + l172: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('-') { + goto l173 + } + position++ + goto l169 + l173: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('_') { goto l174 } position++ - goto l173 + goto l169 l174: - position, tokenIndex = position174, tokenIndex174 + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune(':') { + goto l165 + } + position++ } - add(rulePegText, position170) + l169: + l167: + { + position168, tokenIndex168 := position, tokenIndex + { + position175, tokenIndex175 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l176 + } + position++ + goto l175 + l176: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l177 + } + position++ + goto l175 + l177: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l178 + } + position++ + goto l175 + l178: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('-') { + goto l179 + } + position++ + goto l175 + l179: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('_') { + goto l180 + } + position++ + goto l175 + l180: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune(':') { + goto l168 + } + position++ + } + l175: + goto l167 + l168: + position, tokenIndex = position168, tokenIndex168 + } + add(rulePegText, position166) } { add(ruleAction38, position) } - goto l142 - l169: - position, tokenIndex = position142, tokenIndex142 + goto l123 + l165: + position, tokenIndex = position123, tokenIndex123 { - position177 := position - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l176 - } - position++ + position183 := position + if buffer[position] != rune('"') { + goto l182 } - l180: - l178: - { - position179, tokenIndex179 := position, tokenIndex - { - position186, tokenIndex186 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l187 - } - position++ - goto l186 - l187: - position, tokenIndex = position186, tokenIndex186 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l188 - } - position++ - goto l186 - l188: - position, tokenIndex = position186, tokenIndex186 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l189 - } - position++ - goto l186 - l189: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune('-') { - goto l190 - } - position++ - goto l186 - l190: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune('_') { - goto l191 - } - position++ - goto l186 - l191: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune(':') { - goto l179 - } - position++ - } - l186: - goto l178 - l179: - position, tokenIndex = position179, tokenIndex179 + position++ + if !_rules[ruledoublequotedstring]() { + goto l182 } - add(rulePegText, position177) + if buffer[position] != rune('"') { + goto l182 + } + position++ + add(rulePegText, position183) } { add(ruleAction39, position) } - goto l142 - l176: - position, tokenIndex = position142, tokenIndex142 - { - position194 := position - if buffer[position] != rune('"') { - goto l193 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l193 - } - if buffer[position] != rune('"') { - goto l193 - } - position++ - add(rulePegText, position194) + goto l123 + l182: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('\'') { + goto l121 } + position++ + { + position185 := position + if !_rules[rulesinglequotedstring]() { + goto l121 + } + add(rulePegText, position185) + } + if buffer[position] != rune('\'') { + goto l121 + } + position++ { add(ruleAction40, position) } - goto l142 - l193: - position, tokenIndex = position142, tokenIndex142 - if buffer[position] != rune('\'') { - goto l140 - } - position++ - { - position196 := position - if !_rules[rulesinglequotedstring]() { - goto l140 - } - add(rulePegText, position196) - } - if buffer[position] != rune('\'') { - goto l140 - } - position++ - { - add(ruleAction41, position) - } } - l142: - add(ruleitem, position141) + l123: + add(ruleitem, position122) } return true - l140: - position, tokenIndex = position140, tokenIndex140 + l121: + position, tokenIndex = position121, tokenIndex121 return false }, - /* 14 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ + /* 13 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position199 := position - l200: + position188 := position + l189: { - position201, tokenIndex201 := position, tokenIndex + position190, tokenIndex190 := position, tokenIndex { - position202, tokenIndex202 := position, tokenIndex + position191, tokenIndex191 := position, tokenIndex if buffer[position] != rune('\\') { - goto l203 + goto l192 } position++ if buffer[position] != rune('"') { - goto l203 + goto l192 } position++ - goto l202 - l203: - position, tokenIndex = position202, tokenIndex202 + goto l191 + l192: + position, tokenIndex = position191, tokenIndex191 if buffer[position] != rune('\\') { - goto l204 + goto l193 } position++ if buffer[position] != rune('\\') { - goto l204 + goto l193 } position++ - goto l202 - l204: - position, tokenIndex = position202, tokenIndex202 + goto l191 + l193: + position, tokenIndex = position191, tokenIndex191 { - position205, tokenIndex205 := position, tokenIndex + position194, tokenIndex194 := position, tokenIndex if buffer[position] != rune('"') { - goto l205 + goto l194 } position++ - goto l201 - l205: - position, tokenIndex = position205, tokenIndex205 + goto l190 + l194: + position, tokenIndex = position194, tokenIndex194 } if !matchDot() { - goto l201 + goto l190 } } - l202: - goto l200 - l201: - position, tokenIndex = position201, tokenIndex201 + l191: + goto l189 + l190: + position, tokenIndex = position190, tokenIndex190 } - add(ruledoublequotedstring, position199) + add(ruledoublequotedstring, position188) } return true }, - /* 15 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ + /* 14 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position207 := position - l208: + position196 := position + l197: { - position209, tokenIndex209 := position, tokenIndex + position198, tokenIndex198 := position, tokenIndex { - position210, tokenIndex210 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex if buffer[position] != rune('\\') { - goto l211 + goto l200 } position++ if buffer[position] != rune('\'') { - goto l211 + goto l200 } position++ - goto l210 - l211: - position, tokenIndex = position210, tokenIndex210 + goto l199 + l200: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l212 + goto l201 } position++ if buffer[position] != rune('\\') { - goto l212 + goto l201 } position++ - goto l210 - l212: - position, tokenIndex = position210, tokenIndex210 + goto l199 + l201: + position, tokenIndex = position199, tokenIndex199 { - position213, tokenIndex213 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex if buffer[position] != rune('\'') { - goto l213 + goto l202 } position++ - goto l209 - l213: - position, tokenIndex = position213, tokenIndex213 + goto l198 + l202: + position, tokenIndex = position202, tokenIndex202 } if !matchDot() { - goto l209 + goto l198 } } - l210: - goto l208 - l209: - position, tokenIndex = position209, tokenIndex209 + l199: + goto l197 + l198: + position, tokenIndex = position198, tokenIndex198 } - add(rulesinglequotedstring, position207) + add(rulesinglequotedstring, position196) } return true }, - /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + func() bool { + position203, tokenIndex203 := position, tokenIndex + { + position204 := position + { + position205, tokenIndex205 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l206 + } + position++ + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l203 + } + position++ + } + l205: + l207: + { + position208, tokenIndex208 := position, tokenIndex + { + position209, tokenIndex209 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l210 + } + position++ + goto l209 + l210: + position, tokenIndex = position209, tokenIndex209 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l211 + } + position++ + goto l209 + l211: + position, tokenIndex = position209, tokenIndex209 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l212 + } + position++ + goto l209 + l212: + position, tokenIndex = position209, tokenIndex209 + if buffer[position] != rune('_') { + goto l213 + } + position++ + goto l209 + l213: + position, tokenIndex = position209, tokenIndex209 + if buffer[position] != rune('-') { + goto l208 + } + position++ + } + l209: + goto l207 + l208: + position, tokenIndex = position208, tokenIndex208 + } + add(rulefieldExpr, position204) + } + return true + l203: + position, tokenIndex = position203, tokenIndex203 + return false + }, + /* 16 field <- <(<(fieldExpr / reserved)> Action41)> */ func() bool { position214, tokenIndex214 := position, tokenIndex { position215 := position { - position216, tokenIndex216 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l217 - } - position++ - goto l216 - l217: - position, tokenIndex = position216, tokenIndex216 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l214 - } - position++ - } - l216: - l218: - { - position219, tokenIndex219 := position, tokenIndex + position216 := position { - position220, tokenIndex220 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l221 + position217, tokenIndex217 := position, tokenIndex + if !_rules[rulefieldExpr]() { + goto l218 } - position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l222 + goto l217 + l218: + position, tokenIndex = position217, tokenIndex217 + { + position219 := position + { + position220, tokenIndex220 := position, tokenIndex + if buffer[position] != rune('_') { + goto l221 + } + position++ + if buffer[position] != rune('r') { + goto l221 + } + position++ + if buffer[position] != rune('o') { + goto l221 + } + position++ + if buffer[position] != rune('w') { + goto l221 + } + position++ + goto l220 + l221: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('_') { + goto l222 + } + position++ + if buffer[position] != rune('c') { + goto l222 + } + position++ + if buffer[position] != rune('o') { + goto l222 + } + position++ + if buffer[position] != rune('l') { + goto l222 + } + position++ + goto l220 + l222: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('_') { + goto l223 + } + position++ + if buffer[position] != rune('s') { + goto l223 + } + position++ + if buffer[position] != rune('t') { + goto l223 + } + position++ + if buffer[position] != rune('a') { + goto l223 + } + position++ + if buffer[position] != rune('r') { + goto l223 + } + position++ + if buffer[position] != rune('t') { + goto l223 + } + position++ + goto l220 + l223: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('_') { + goto l224 + } + position++ + if buffer[position] != rune('e') { + goto l224 + } + position++ + if buffer[position] != rune('n') { + goto l224 + } + position++ + if buffer[position] != rune('d') { + goto l224 + } + position++ + goto l220 + l224: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('_') { + goto l225 + } + position++ + if buffer[position] != rune('t') { + goto l225 + } + position++ + if buffer[position] != rune('i') { + goto l225 + } + position++ + if buffer[position] != rune('m') { + goto l225 + } + position++ + if buffer[position] != rune('e') { + goto l225 + } + position++ + if buffer[position] != rune('s') { + goto l225 + } + position++ + if buffer[position] != rune('t') { + goto l225 + } + position++ + if buffer[position] != rune('a') { + goto l225 + } + position++ + if buffer[position] != rune('m') { + goto l225 + } + position++ + if buffer[position] != rune('p') { + goto l225 + } + position++ + goto l220 + l225: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('_') { + goto l214 + } + position++ + if buffer[position] != rune('f') { + goto l214 + } + position++ + if buffer[position] != rune('i') { + goto l214 + } + position++ + if buffer[position] != rune('e') { + goto l214 + } + position++ + if buffer[position] != rune('l') { + goto l214 + } + position++ + if buffer[position] != rune('d') { + goto l214 + } + position++ + } + l220: + add(rulereserved, position219) } - position++ - goto l220 - l222: - position, tokenIndex = position220, tokenIndex220 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l223 - } - position++ - goto l220 - l223: - position, tokenIndex = position220, tokenIndex220 - if buffer[position] != rune('_') { - goto l224 - } - position++ - goto l220 - l224: - position, tokenIndex = position220, tokenIndex220 - if buffer[position] != rune('-') { - goto l219 - } - position++ } - l220: - goto l218 - l219: - position, tokenIndex = position219, tokenIndex219 + l217: + add(rulePegText, position216) } - add(rulefieldExpr, position215) + { + add(ruleAction41, position) + } + add(rulefield, position215) } return true l214: position, tokenIndex = position214, tokenIndex214 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action42)> */ + /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ + nil, + /* 18 posfield <- <( Action42)> */ func() bool { - position225, tokenIndex225 := position, tokenIndex + position228, tokenIndex228 := position, tokenIndex { - position226 := position + position229 := position { - position227 := position - { - position228, tokenIndex228 := position, tokenIndex - if !_rules[rulefieldExpr]() { - goto l229 - } + position230 := position + if !_rules[rulefieldExpr]() { goto l228 - l229: - position, tokenIndex = position228, tokenIndex228 - { - position230 := position - { - position231, tokenIndex231 := position, tokenIndex - if buffer[position] != rune('_') { - goto l232 - } - position++ - if buffer[position] != rune('r') { - goto l232 - } - position++ - if buffer[position] != rune('o') { - goto l232 - } - position++ - if buffer[position] != rune('w') { - goto l232 - } - position++ - goto l231 - l232: - position, tokenIndex = position231, tokenIndex231 - if buffer[position] != rune('_') { - goto l233 - } - position++ - if buffer[position] != rune('c') { - goto l233 - } - position++ - if buffer[position] != rune('o') { - goto l233 - } - position++ - if buffer[position] != rune('l') { - goto l233 - } - position++ - goto l231 - l233: - position, tokenIndex = position231, tokenIndex231 - if buffer[position] != rune('_') { - goto l234 - } - position++ - if buffer[position] != rune('s') { - goto l234 - } - position++ - if buffer[position] != rune('t') { - goto l234 - } - position++ - if buffer[position] != rune('a') { - goto l234 - } - position++ - if buffer[position] != rune('r') { - goto l234 - } - position++ - if buffer[position] != rune('t') { - goto l234 - } - position++ - goto l231 - l234: - position, tokenIndex = position231, tokenIndex231 - if buffer[position] != rune('_') { - goto l235 - } - position++ - if buffer[position] != rune('e') { - goto l235 - } - position++ - if buffer[position] != rune('n') { - goto l235 - } - position++ - if buffer[position] != rune('d') { - goto l235 - } - position++ - goto l231 - l235: - position, tokenIndex = position231, tokenIndex231 - if buffer[position] != rune('_') { - goto l236 - } - position++ - if buffer[position] != rune('t') { - goto l236 - } - position++ - if buffer[position] != rune('i') { - goto l236 - } - position++ - if buffer[position] != rune('m') { - goto l236 - } - position++ - if buffer[position] != rune('e') { - goto l236 - } - position++ - if buffer[position] != rune('s') { - goto l236 - } - position++ - if buffer[position] != rune('t') { - goto l236 - } - position++ - if buffer[position] != rune('a') { - goto l236 - } - position++ - if buffer[position] != rune('m') { - goto l236 - } - position++ - if buffer[position] != rune('p') { - goto l236 - } - position++ - goto l231 - l236: - position, tokenIndex = position231, tokenIndex231 - if buffer[position] != rune('_') { - goto l225 - } - position++ - if buffer[position] != rune('f') { - goto l225 - } - position++ - if buffer[position] != rune('i') { - goto l225 - } - position++ - if buffer[position] != rune('e') { - goto l225 - } - position++ - if buffer[position] != rune('l') { - goto l225 - } - position++ - if buffer[position] != rune('d') { - goto l225 - } - position++ - } - l231: - add(rulereserved, position230) - } } - l228: - add(rulePegText, position227) + add(rulePegText, position230) } { add(ruleAction42, position) } - add(rulefield, position226) + add(ruleposfield, position229) } return true - l225: - position, tokenIndex = position225, tokenIndex225 + l228: + position, tokenIndex = position228, tokenIndex228 return false }, - /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ - nil, - /* 19 posfield <- <( Action43)> */ + /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position239, tokenIndex239 := position, tokenIndex + position232, tokenIndex232 := position, tokenIndex { - position240 := position + position233 := position { - position241 := position - if !_rules[rulefieldExpr]() { - goto l239 - } - add(rulePegText, position241) - } - { - add(ruleAction43, position) - } - add(ruleposfield, position240) - } - return true - l239: - position, tokenIndex = position239, tokenIndex239 - return false - }, - /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ - func() bool { - position243, tokenIndex243 := position, tokenIndex - { - position244 := position - { - position245, tokenIndex245 := position, tokenIndex + position234, tokenIndex234 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l246 + goto l235 } position++ - l247: + l236: { - position248, tokenIndex248 := position, tokenIndex + position237, tokenIndex237 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l248 + goto l237 } position++ - goto l247 - l248: - position, tokenIndex = position248, tokenIndex248 + goto l236 + l237: + position, tokenIndex = position237, tokenIndex237 } - goto l245 - l246: - position, tokenIndex = position245, tokenIndex245 + goto l234 + l235: + position, tokenIndex = position234, tokenIndex234 if buffer[position] != rune('0') { - goto l243 + goto l232 } position++ } - l245: - add(ruleuint, position244) + l234: + add(ruleuint, position233) } return true - l243: - position, tokenIndex = position243, tokenIndex243 + l232: + position, tokenIndex = position232, tokenIndex232 return false }, - /* 21 col <- <(( Action44) / ('\'' '\'' Action45) / ('"' '"' Action46))> */ + /* 20 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ func() bool { - position249, tokenIndex249 := position, tokenIndex + position238, tokenIndex238 := position, tokenIndex { - position250 := position + position239 := position { - position251, tokenIndex251 := position, tokenIndex + position240, tokenIndex240 := position, tokenIndex { - position253 := position + position242 := position if !_rules[ruleuint]() { - goto l252 + goto l241 } - add(rulePegText, position253) + add(rulePegText, position242) } + { + add(ruleAction43, position) + } + goto l240 + l241: + position, tokenIndex = position240, tokenIndex240 + if buffer[position] != rune('\'') { + goto l244 + } + position++ + { + position245 := position + if !_rules[rulesinglequotedstring]() { + goto l244 + } + add(rulePegText, position245) + } + if buffer[position] != rune('\'') { + goto l244 + } + position++ { add(ruleAction44, position) } - goto l251 - l252: - position, tokenIndex = position251, tokenIndex251 - if buffer[position] != rune('\'') { - goto l255 + goto l240 + l244: + position, tokenIndex = position240, tokenIndex240 + if buffer[position] != rune('"') { + goto l238 } position++ { - position256 := position - if !_rules[rulesinglequotedstring]() { - goto l255 + position247 := position + if !_rules[ruledoublequotedstring]() { + goto l238 } - add(rulePegText, position256) + add(rulePegText, position247) } - if buffer[position] != rune('\'') { - goto l255 + if buffer[position] != rune('"') { + goto l238 } position++ { add(ruleAction45, position) } - goto l251 - l255: - position, tokenIndex = position251, tokenIndex251 - if buffer[position] != rune('"') { - goto l249 - } - position++ - { - position258 := position - if !_rules[ruledoublequotedstring]() { - goto l249 - } - add(rulePegText, position258) - } - if buffer[position] != rune('"') { - goto l249 - } - position++ - { - add(ruleAction46, position) - } } - l251: - add(rulecol, position250) + l240: + add(rulecol, position239) } return true - l249: - position, tokenIndex = position249, tokenIndex249 + l238: + position, tokenIndex = position238, tokenIndex238 return false }, - /* 22 row <- <(( Action47) / ('\'' '\'' Action48) / ('"' '"' Action49))> */ + /* 21 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ nil, - /* 23 open <- <('(' sp)> */ + /* 22 open <- <('(' sp)> */ + func() bool { + position250, tokenIndex250 := position, tokenIndex + { + position251 := position + if buffer[position] != rune('(') { + goto l250 + } + position++ + if !_rules[rulesp]() { + goto l250 + } + add(ruleopen, position251) + } + return true + l250: + position, tokenIndex = position250, tokenIndex250 + return false + }, + /* 23 close <- <(')' sp)> */ + func() bool { + position252, tokenIndex252 := position, tokenIndex + { + position253 := position + if buffer[position] != rune(')') { + goto l252 + } + position++ + if !_rules[rulesp]() { + goto l252 + } + add(ruleclose, position253) + } + return true + l252: + position, tokenIndex = position252, tokenIndex252 + return false + }, + /* 24 sp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position255 := position + l256: + { + position257, tokenIndex257 := position, tokenIndex + { + position258, tokenIndex258 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l259 + } + position++ + goto l258 + l259: + position, tokenIndex = position258, tokenIndex258 + if buffer[position] != rune('\t') { + goto l260 + } + position++ + goto l258 + l260: + position, tokenIndex = position258, tokenIndex258 + if buffer[position] != rune('\n') { + goto l257 + } + position++ + } + l258: + goto l256 + l257: + position, tokenIndex = position257, tokenIndex257 + } + add(rulesp, position255) + } + return true + }, + /* 25 comma <- <(sp ',' sp)> */ func() bool { position261, tokenIndex261 := position, tokenIndex { position262 := position - if buffer[position] != rune('(') { + if !_rules[rulesp]() { + goto l261 + } + if buffer[position] != rune(',') { goto l261 } position++ if !_rules[rulesp]() { goto l261 } - add(ruleopen, position262) + add(rulecomma, position262) } return true l261: position, tokenIndex = position261, tokenIndex261 return false }, - /* 24 close <- <(')' sp)> */ - func() bool { - position263, tokenIndex263 := position, tokenIndex - { - position264 := position - if buffer[position] != rune(')') { - goto l263 - } - position++ - if !_rules[rulesp]() { - goto l263 - } - add(ruleclose, position264) - } - return true - l263: - position, tokenIndex = position263, tokenIndex263 - return false - }, - /* 25 sp <- <(' ' / '\t' / '\n')*> */ + /* 26 lbrack <- <('[' sp)> */ + nil, + /* 27 rbrack <- <(sp ']' sp)> */ + nil, + /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { + position265, tokenIndex265 := position, tokenIndex { position266 := position - l267: { - position268, tokenIndex268 := position, tokenIndex + position267, tokenIndex267 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l268 + } + position++ + goto l267 + l268: + position, tokenIndex = position267, tokenIndex267 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l265 + } + position++ + } + l267: + l269: + { + position270, tokenIndex270 := position, tokenIndex { - position269, tokenIndex269 := position, tokenIndex - if buffer[position] != rune(' ') { + position271, tokenIndex271 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l272 + } + position++ + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l273 + } + position++ + goto l271 + l273: + position, tokenIndex = position271, tokenIndex271 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l270 } position++ - goto l269 - l270: - position, tokenIndex = position269, tokenIndex269 - if buffer[position] != rune('\t') { - goto l271 - } - position++ - goto l269 - l271: - position, tokenIndex = position269, tokenIndex269 - if buffer[position] != rune('\n') { - goto l268 - } - position++ } - l269: - goto l267 - l268: - position, tokenIndex = position268, tokenIndex268 + l271: + goto l269 + l270: + position, tokenIndex = position270, tokenIndex270 } - add(rulesp, position266) + add(ruleIDENT, position266) } return true - }, - /* 26 comma <- <(sp ',' sp)> */ - func() bool { - position272, tokenIndex272 := position, tokenIndex - { - position273 := position - if !_rules[rulesp]() { - goto l272 - } - if buffer[position] != rune(',') { - goto l272 - } - position++ - if !_rules[rulesp]() { - goto l272 - } - add(rulecomma, position273) - } - return true - l272: - position, tokenIndex = position272, tokenIndex272 + l265: + position, tokenIndex = position265, tokenIndex265 return false }, - /* 27 lbrack <- <('[' sp)> */ - nil, - /* 28 rbrack <- <(sp ']' sp)> */ - nil, - /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ - nil, - /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + /* 29 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position278 := position + position275 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if buffer[position] != rune('-') { - goto l277 + goto l274 } position++ { - position279, tokenIndex279 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex if buffer[position] != rune('0') { - goto l280 - } - position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 - if buffer[position] != rune('1') { goto l277 } position++ + goto l276 + l277: + position, tokenIndex = position276, tokenIndex276 + if buffer[position] != rune('1') { + goto l274 + } + position++ } - l279: + l276: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if buffer[position] != rune('-') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if buffer[position] != rune('T') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if buffer[position] != rune(':') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l274 } position++ - add(ruletimestampbasicfmt, position278) + add(ruletimestampbasicfmt, position275) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l274: + position, tokenIndex = position274, tokenIndex274 return false }, - /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + /* 30 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position281, tokenIndex281 := position, tokenIndex + position278, tokenIndex278 := position, tokenIndex { - position282 := position + position279 := position { - position283, tokenIndex283 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex if buffer[position] != rune('"') { - goto l284 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l284 - } - if buffer[position] != rune('"') { - goto l284 - } - position++ - goto l283 - l284: - position, tokenIndex = position283, tokenIndex283 - if buffer[position] != rune('\'') { - goto l285 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l285 - } - if buffer[position] != rune('\'') { - goto l285 - } - position++ - goto l283 - l285: - position, tokenIndex = position283, tokenIndex283 - if !_rules[ruletimestampbasicfmt]() { goto l281 } + position++ + { + position282 := position + if !_rules[ruletimestampbasicfmt]() { + goto l281 + } + add(rulePegText, position282) + } + if buffer[position] != rune('"') { + goto l281 + } + position++ + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 + if buffer[position] != rune('\'') { + goto l283 + } + position++ + { + position284 := position + if !_rules[ruletimestampbasicfmt]() { + goto l283 + } + add(rulePegText, position284) + } + if buffer[position] != rune('\'') { + goto l283 + } + position++ + goto l280 + l283: + position, tokenIndex = position280, tokenIndex280 + { + position285 := position + if !_rules[ruletimestampbasicfmt]() { + goto l278 + } + add(rulePegText, position285) + } } - l283: - add(ruletimestampfmt, position282) + l280: + add(ruletimestampfmt, position279) } return true - l281: - position, tokenIndex = position281, tokenIndex281 + l278: + position, tokenIndex = position278, tokenIndex278 return false }, - /* 32 timestamp <- <( Action50)> */ + /* 31 timestamp <- <( Action49)> */ nil, - /* 34 Action0 <- <{p.startCall("Set")}> */ + /* 33 Action0 <- <{p.startCall("Set")}> */ nil, - /* 35 Action1 <- <{p.endCall()}> */ + /* 34 Action1 <- <{p.endCall()}> */ nil, - /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 35 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 37 Action3 <- <{p.endCall()}> */ + /* 36 Action3 <- <{p.endCall()}> */ nil, - /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + /* 37 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, - /* 39 Action5 <- <{p.endCall()}> */ + /* 38 Action5 <- <{p.endCall()}> */ nil, - /* 40 Action6 <- <{p.startCall("Clear")}> */ + /* 39 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 41 Action7 <- <{p.endCall()}> */ + /* 40 Action7 <- <{p.endCall()}> */ nil, - /* 42 Action8 <- <{p.startCall("ClearRow")}> */ + /* 41 Action8 <- <{p.startCall("ClearRow")}> */ nil, - /* 43 Action9 <- <{p.endCall()}> */ + /* 42 Action9 <- <{p.endCall()}> */ nil, - /* 44 Action10 <- <{p.startCall("Store")}> */ + /* 43 Action10 <- <{p.startCall("Store")}> */ nil, - /* 45 Action11 <- <{p.endCall()}> */ + /* 44 Action11 <- <{p.endCall()}> */ nil, - /* 46 Action12 <- <{p.startCall("TopN")}> */ + /* 45 Action12 <- <{p.startCall("TopN")}> */ nil, - /* 47 Action13 <- <{p.endCall()}> */ - nil, - /* 48 Action14 <- <{p.startCall("Range")}> */ - nil, - /* 49 Action15 <- <{p.endCall()}> */ + /* 46 Action13 <- <{p.endCall()}> */ nil, nil, - /* 51 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 52 Action17 <- <{ p.endCall() }> */ + /* 49 Action15 <- <{ p.endCall() }> */ nil, - /* 53 Action18 <- <{ p.addBTWN() }> */ + /* 50 Action16 <- <{ p.addBTWN() }> */ nil, - /* 54 Action19 <- <{ p.addLTE() }> */ + /* 51 Action17 <- <{ p.addLTE() }> */ nil, - /* 55 Action20 <- <{ p.addGTE() }> */ + /* 52 Action18 <- <{ p.addGTE() }> */ nil, - /* 56 Action21 <- <{ p.addEQ() }> */ + /* 53 Action19 <- <{ p.addEQ() }> */ nil, - /* 57 Action22 <- <{ p.addNEQ() }> */ + /* 54 Action20 <- <{ p.addNEQ() }> */ nil, - /* 58 Action23 <- <{ p.addLT() }> */ + /* 55 Action21 <- <{ p.addLT() }> */ nil, - /* 59 Action24 <- <{ p.addGT() }> */ + /* 56 Action22 <- <{ p.addGT() }> */ nil, - /* 60 Action25 <- <{p.startConditional()}> */ + /* 57 Action23 <- <{p.startConditional()}> */ nil, - /* 61 Action26 <- <{p.endConditional()}> */ + /* 58 Action24 <- <{p.endConditional()}> */ nil, - /* 62 Action27 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 63 Action28 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action26 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action29 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 65 Action30 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action28 <- <{ p.startList() }> */ nil, - /* 66 Action31 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action29 <- <{ p.endList() }> */ nil, - /* 67 Action32 <- <{ p.startList() }> */ + /* 64 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 68 Action33 <- <{ p.endList() }> */ + /* 65 Action31 <- <{ p.addVal(true) }> */ nil, - /* 69 Action34 <- <{ p.addVal(nil) }> */ + /* 66 Action32 <- <{ p.addVal(false) }> */ nil, - /* 70 Action35 <- <{ p.addVal(true) }> */ + /* 67 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 71 Action36 <- <{ p.addVal(false) }> */ + /* 68 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 72 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 73 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action36 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 74 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action37 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 75 Action40 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 72 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action39 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 77 Action42 <- <{ p.addField(buffer[begin:end]) }> */ + /* 74 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 78 Action43 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 75 Action41 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 79 Action44 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 76 Action42 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 80 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 77 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 81 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 82 Action47 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 79 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 80 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 84 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 81 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ nil, - /* 85 Action50 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 82 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 83 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 2025120e9..54f73b809 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -8,7 +8,7 @@ import ( func TestPEG(t *testing.T) { p := PQL{Buffer: ` -SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} p.Init() err := p.Parse() if err != nil { @@ -202,51 +202,59 @@ func TestPEGWorking(t *testing.T) { ncalls: 1}, { name: "RangeLT", - input: "Range(a < 4)", + input: "Row(a < 4)", ncalls: 1}, { name: "RangeGT", - input: "Range(a > 4)", + input: "Row(a > 4)", ncalls: 1}, { name: "RangeLTE", - input: "Range(a <= 4)", + input: "Row(a <= 4)", ncalls: 1}, { name: "RangeGTE", - input: "Range(a >= 4)", + input: "Row(a >= 4)", ncalls: 1}, { name: "RangeEQ", - input: "Range(a == 4)", + input: "Row(a == 4)", ncalls: 1}, { name: "RangeNEQ", - input: "Range(a != null)", + input: "Row(a != null)", ncalls: 1}, { name: "RangeLTLT", - input: "Range(4 < a < 9)", + input: "Row(4 < a < 9)", ncalls: 1}, { name: "RangeLTLTE", - input: "Range(4 < a <= 9)", + input: "Row(4 < a <= 9)", ncalls: 1}, { name: "RangeLTELT", - input: "Range(4 <= a < 9)", + input: "Row(4 <= a < 9)", ncalls: 1}, { name: "RangeLTELTE", - input: "Range(4 <= a <= 9)", + input: "Row(4 <= a <= 9)", ncalls: 1}, { name: "RangeTime", - input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)", + input: "Row(a=4, from=2010-07-04T00:00, to=2010-08-04T00:00)", ncalls: 1}, { name: "RangeTimeQuotes", - input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`, + input: `Row(a=4, from='2010-07-04T00:00', to="2010-08-04T00:00")`, + ncalls: 1}, + { + name: "RangeTimeFromQuotes", + input: `Row(a=4, from='2010-07-04T00:00')`, + ncalls: 1}, + { + name: "RangeTimeToQuotes", + input: `Row(a=4, to="2010-08-04T00:00")`, ncalls: 1}, { name: "Dashed Frame", @@ -302,10 +310,10 @@ func TestPEGErrors(t *testing.T) { input: "Clear(9)"}, { name: "RangeTimeGT", - input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, + input: "Row(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, { name: "RangeTimeOneStamp", - input: "Range(a=4, 2010-07-04T00:00)"}, + input: "Row(a=4, 2010-07-04T00:00)"}, } for i, test := range tests { @@ -423,9 +431,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeEQ", - call: "Range(a==7)", + call: "Row(a==7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: EQ, @@ -435,9 +443,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLT", - call: "Range(a<7)", + call: "Row(a<7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: LT, @@ -447,9 +455,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTE", - call: "Range(a<=7)", + call: "Row(a<=7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: LTE, @@ -459,9 +467,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeGTE", - call: "Range(a>=7)", + call: "Row(a>=7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: GTE, @@ -471,9 +479,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeGT", - call: "Range(a>7)", + call: "Row(a>7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: GT, @@ -483,9 +491,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeNEQ", - call: "Range(a!=null)", + call: "Row(a!=null)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: NEQ, @@ -495,9 +503,33 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTELT", - call: "Range(4 <= a < 9)", + call: "Row(4 <= a < 9)", exp: &Call{ - Name: "Range", + Name: "Row", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(4), int64(8)}, + }, + }, + }}, + { + name: "RangeLTLT", + call: "Row(4 < a < 9)", + exp: &Call{ + Name: "Row", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(8)}, + }, + }, + }}, + { + name: "RangeLTELTE", + call: "Row(4 <= a <= 9)", + exp: &Call{ + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, @@ -506,10 +538,10 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { - name: "RangeLTLT", - call: "Range(4 < a < 9)", + name: "RangeLTLTE", + call: "Row(4 < a <= 9)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, @@ -517,30 +549,6 @@ func TestPQLDeepEquality(t *testing.T) { }, }, }}, - { - name: "RangeLTELTE", - call: "Range(4 <= a <= 9)", - exp: &Call{ - Name: "Range", - Args: map[string]interface{}{ - "a": &Condition{ - Op: BETWEEN, - Value: []interface{}{int64(4), int64(10)}, - }, - }, - }}, - { - name: "RangeLTLTE", - call: "Range(4 < a <= 9)", - exp: &Call{ - Name: "Range", - Args: map[string]interface{}{ - "a": &Condition{ - Op: BETWEEN, - Value: []interface{}{int64(5), int64(10)}, - }, - }, - }}, { name: "Sum", call: "Sum(field=f)", @@ -612,6 +620,43 @@ func TestPQLDeepEquality(t *testing.T) { }, }, }}, + { + name: "GroupBy", + call: "GroupBy(Rows(), filter=Row(a=1))", + exp: &Call{ + Name: "GroupBy", + Args: map[string]interface{}{ + "filter": &Call{ + Name: "Row", + Args: map[string]interface{}{ + "a": int64(1), + }, + }, + }, + Children: []*Call{ + {Name: "Rows"}, + }, + }}, + { + name: "GroupByFilterRangeLTLT", + call: "GroupBy(Rows(), filter=Row(4 < a < 9))", + exp: &Call{ + Name: "GroupBy", + Args: map[string]interface{}{ + "filter": &Call{ + Name: "Row", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(8)}, + }, + }, + }, + }, + Children: []*Call{ + {Name: "Rows"}, + }, + }}, } for i, test := range tests { diff --git a/roaring/containers.go b/roaring/containers.go index 3fe0814cc..b9928823a 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -156,6 +156,12 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found return &sliceIterator{e: sc, i: i}, found } +func (sc *sliceContainers) Repair() { + for _, c := range sc.containers { + c.Repair() + } +} + type sliceIterator struct { e *sliceContainers i int diff --git a/roaring/roaring.go b/roaring/roaring.go index 8c3cbc52e..7624b9f13 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,8 +98,12 @@ type Containers interface { Count() uint64 - //Reset will clear the containers collection to allow for recycling during snapshot + // Reset clears the containers collection to allow for recycling during snapshot Reset() + + // Repair will repair the cardinality of any containers whose cardinality were corrupted + // due to optimized operations. + Repair() } type ContainerIterator interface { @@ -395,10 +399,26 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return output } -// Union returns the bitwise union of b and other. -func (b *Bitmap) Union(other *Bitmap) *Bitmap { +// Union returns the bitwise union of b and others as a new bitmap. +func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { output := NewBitmap() + if len(others) == 1 { + b.unionIntoTargetSingle(output, others[0]) + return output + } + output.UnionInPlace(b) + output.UnionInPlace(others...) + return output +} + +// UnionInPlace returns the bitwise union of b and others, modifying +// b in place. +func (b *Bitmap) UnionInPlace(others ...*Bitmap) { + b.unionInPlace(others...) +} + +func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -406,21 +426,265 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + target.Containers.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.Containers.Put(kj, cj.Clone()) + target.Containers.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.Containers.Put(ki, union(ci, cj)) + target.Containers.Put(ki, union(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } - return output +} + +// unionIntoTarget stores the union of b and others into target. b and others will +// be left unchanged (unless one of them is also target), but target will be modified +// in place. +// +// This function performs an n-way union of n bitmaps. It performs this in an +// optimized manner looping through all the bitmaps and performing unions one +// container at a time. As a result, instead of generating many intermediary +// containers for each union operation for a given container key, only one +// new container needs to be allocated (or re-used) regardless of how many bitmaps +// participate in the union. This significantly reduces allocations. In addition, +// because we perform the unions one container at a time across all the bitmaps, we +// can calculate summary statistics that allow us to make more efficient decisions +// up front. For example, imagine trying to perform a union across the following three +// bitsets: +// +// 1. Bitmap A: Single array container at key 0 with 400 values in it. +// 2. Bitmap B: Single array container at key 0 with 500 values in it. +// 3. Bitmap C: Single array container at key 0 with 3500 values in it. +// +// Naive approach: +// +// 1. Perform union of bitmap A and B, container by container +// a. 400 + 500 < ArrayMaxSize so likely we will choose to allocate a new array +// container and then perform a unionArrayArray operation to merge the two +// arrays into the new array container. +// 2. Perform a union of the bitmap generated in the step above with bitmap C. +// 900 + 3500 > ArrayMaxSize so we will need to upgrade to a bitset container which +// we will have to allocate, and then we will have to perform two unions into the +// new bitmap container: one for the array container generated in the previous step, +// and one for the bitset container in bitmap C. +// +// Approach taken by this function: +// +// 1. Detect that bitmaps A, B, and C all have containers for key 0. +// 2. Estimate the resulting cardinality of the union of all their containers to be +// 400 + 500 + 3500 > ArrayMaxSize and decide upfront to use a bitset for the target +// container. Note that this is just an approximation of the final cardinality and can +// be off by a wide margin if there is a lot of overlap between containers, but that is +// fine, we'll still get the same result at the end, we'll just be more biased towards +// using bitmap containers will still being able to use array containers when all the +// cardinalities are small. +// 3. Union the containers from bitmaps A, B, and C into the new bitset container directly +// using fast bitwise operations. +// +// In the naive approach, we had to allocate two containers, whereas in the optimized approach +// we only had to allocate one container, and we also had to perform less union operations. This +// example is simplistic, but the impact in terms of CPU cycles and memory allocations achieved +// by using the optimized alogorithm when unioning many large bitmaps can be huge. +// +// An additional optimization that this function makes is that it recognizes that even when +// CPU support is present, performing the popcount() operation isn't free. Imagine a scenario +// where 10 bitset containers are being unioned together one after the next. If every +// bitset<->bitset union operation needs to keep the containers' cardinality up to date, then +// the algorithm will waste a lot of time performing intermediary popcount() operations that +// will immediately be invalidated by the next union operation. As a result, we allow the cardinality +// of containers to degrade when we perform the in-place union operations, and then when the algorithm +// completes we "repair" all the containers by performing the popcount() operation one time. This means +// that we only ever have to do O(1) popcount operations per container instead of O(n) where n is the +// number of containers with the same key that are being unioned together. +// +// The algorithm works by iterating through all of the containers in all of the bitmaps concurrently. +// At every "tick" of the outermost loop, we increment our pointer into the bitmaps list of containers +// by 1 (if we haven't reached the end of the containers for that bitmap.) +// +// We then loop through all of the "current" values(containers) for all of the bitmaps +// and for each container with a specific key that we encounter, we scan forward to see if any of the +// other bitmaps have a container for the same key. If so, we calculate some summary statistics and +// then use that information to make a decision about how to union all of the containers with the same +// key together, perform the union, mark the unioned containers as "handled" and then move on to the next +// batch of containers that share the same key. +// +// We repeat this process until every single bitmaps current container has been "handled". Then we start the +// outer loop over again and the process repeats until we've iterated through every container in every bitmap +// and unioned everything into a single target bitmap. +// +// The diagram below shows the iteration state of four different bitmaps as the algorithm progresses them. +// The diagrams should BE interpreted from left -> right, top -> bottom. The X's represent a container in +// the bitmap at a specific key, ^ symbol represents the bitmaps current container iteration position, +// and the - symbol represents a container that is at the current iteration position, but has been marked as "handled". +// +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| +// ^ | _ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 2 |_______X________X______X___| | |_______X_______________X___| | |_______X_______________X___| +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| +// ^ | _ | +// ------------------------------------------------------------------------------------------------------------------------ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| +// _ | ^ | _ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 2 |_______X_______________X___| | |_______X_______________X___| | |_______X_______________X___| +// _ | ^ | ^ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| +// _ | | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| +// _ +func (b *Bitmap) unionInPlace(others ...*Bitmap) { + var ( + requiredSliceSize = len(others) + // To avoid having to allocate a slice everytime, if the number of bitmaps + // being unioned is small enough we can just use this stack-allocated array. + staticHandledIters = [20]handledIter{} + bitmapIters handledIters + target = b + ) + + if requiredSliceSize <= 20 { + bitmapIters = staticHandledIters[:0] + } else { + bitmapIters = make(handledIters, 0, requiredSliceSize) + } + + for _, other := range others { + otherIter, _ := other.Containers.Iterator(0) + if otherIter.Next() { + bitmapIters = append(bitmapIters, handledIter{ + iter: otherIter, + hasNext: true, + handled: false, + }) + } + } + + // Loop until we've exhausted every iter. + hasNext := true + for hasNext { + // Loop until every iters current value has been handled. + for i, iIter := range bitmapIters { + if !iIter.hasNext || iIter.handled { + // Either we've exhausted this iter (it has no more containers), or + // we've already handled the current container by unioning it with + // one of the containers we encountered earlier. + continue + } + + var ( + iKey, iContainer = iIter.iter.Value() + // Summary statistics about all the containers in the other bitmaps + // that share the same key so we can make smarter union strategy + // decisions later. Note that we slice to [i:] not [i+1:] because we + // want to include the current containers information in the stats. + summaryStats = bitmapIters[i:].calculateSummaryStats(iKey) + ) + + if summaryStats.isOnlyContainerWithKey { + // TODO(rartoul): We can avoid these clones if we can determine + // if the container is coming from an immutable bitmap and we + // know that we can mark the target bitmap as immutable as well. + target.Containers.Put(iKey, iContainer.Clone()) + bitmapIters[i].handled = true + continue + } + + // There was more than one container across the bitmaps with key iKey + // so we need to calculate a union. + if summaryStats.hasMaxRange { + // One (or more) of the containers represented the maximum possible + // range that a container can store, so instead of calculating a + // union we can generate an RLE container that represents the entire + // range. + container := &Container{ + runs: []interval16{{start: 0, last: maxContainerVal}}, + containerType: containerRun, + n: maxContainerVal + 1, + } + target.Containers.Put(iKey, container) + bitmapIters.markItersWithCurrentKeyAsHandled(i, iKey) + continue + } + + // Use a bitmap container for the target bitmap, and union everything + // into it. + // + // TODO(rartoul): Add another conditional case for n < ArrayMaxSize + // (or some fraction of that value) to avoid allocating expensive + // bitmaps when unioning many low-density array containers, but this + // will require writing a union in place algorithm for an array container + // that accepts multiple different containers to union into it for + // efficiency. + container := target.Containers.Get(iKey) + // If target already has a bitmap container for iKey then we can reuse that, + // otherwise we have to allocate a new one or convert the existing container + // into a bitmap container. + if container == nil { + buf := make([]uint64, bitmapN) + ob := buf[:bitmapN] + container = &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } + } else if container.isArray() { + container.arrayToBitmap() + } else if container.isRun() { + container.runToBitmap() + } + + // Once we've acquired a bitmap container (either by reusing the existing one + // or allocating a new one) then the last step is to iterate through all the + // other containers to see which ones have the same key, and union all of them + // into the target bitmap container. Only need to loop starting from i because + // anything previous to that has already been handled. + for j := i; j < len(bitmapIters); j++ { + jIter := bitmapIters[j] + jKey, jContainer := jIter.iter.Value() + + if iKey == jKey { + if jContainer.isArray() { + unionBitmapArrayInPlace(container, jContainer) + } else if jContainer.isRun() { + unionBitmapRunInPlace(container, jContainer) + } else { + unionBitmapBitmapInPlace(container, jContainer) + } + bitmapIters[j].handled = true + } + } + + // Now that we've calculated a container that is a union of all the containers + // with the same key across all the bitmaps, we store it in the list of containers + // for the target. + target.Containers.Put(iKey, container) + } + + hasNext = bitmapIters.next() + } + + // Performing the popcount() operation with every union is wasteful because + // its likely the value will be invalidated by the next union operation. As + // a result, when we're performing all of our in-place unions we allow the value of + // n (container cardinality) to fall out of sync, and then at the very end we perform + // a "Repair" to recalculate all the container values. That way we never popcount() + // an entire bitmap container more than once per bulk union operation. + target.Containers.Repair() } // Difference returns the difference of b and other. @@ -1850,6 +2114,28 @@ func (c *Container) check() error { return a } +// Repair repairs the cardinality of c if it has been corrupted by +// optimized operations. +func (c *Container) Repair() { + if c.isBitmap() { + c.bitmapRepair() + } +} + +func (c *Container) bitmapRepair() { + n := int32(0) + // Manually unroll loop to make it a little faster. + // TODO(rartoul): Can probably make this a few x faster using + // SIMD instructions. + for i := 0; i < bitmapN; i += 4 { + n += int32(popcount(c.bitmap[i])) + n += int32(popcount(c.bitmap[i+1])) + n += int32(popcount(c.bitmap[i+2])) + n += int32(popcount(c.bitmap[i+3])) + } + c.n = n +} + // containerInfo represents a point-in-time snapshot of container stats. type containerInfo struct { Key uint64 // container key @@ -2410,6 +2696,15 @@ func unionBitmapRun(a, b *Container) *Container { return output } +// unions the run b into the bitmap a, mutating a in place. The n value of +// a will need to be repaired after the fact. +func unionBitmapRunInPlace(a, b *Container) { + statsHit("union/BitmapRun") + for j := 0; j < len(b.runs); j++ { + a.bitmapSetRangeIgnoreN(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + } +} + const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j) (c must be a bitmap container) @@ -2435,6 +2730,25 @@ func (c *Container) bitmapSetRange(i, j uint64) { } } +// sets all bits in [i, j) (c must be a bitmap container) without updating +// the value of n, meaning it will need to be repaired after the fact. +func (c *Container) bitmapSetRangeIgnoreN(i, j uint64) { + x := i >> 6 + y := (j - 1) >> 6 + var X uint64 = maxBitmap << (i % 64) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + + if x == y { + c.bitmap[x] |= (X & Y) + } else { + c.bitmap[x] |= X + for i := x + 1; i < y; i++ { + c.bitmap[i] = maxBitmap + } + c.bitmap[y] |= Y + } +} + // xor's all bits in [i, j) with all true (c must be a bitmap container). func (c *Container) bitmapXorRange(i, j uint64) { x := i >> 6 @@ -2529,6 +2843,14 @@ func unionArrayBitmap(a, b *Container) *Container { return output } +// unions array b into bitmap a, mutating a in place. The n value +// of a will need to be repaired after the fact. +func unionBitmapArrayInPlace(a, b *Container) { + for _, v := range b.array { + a.bitmap[v>>6] |= (uint64(1) << (v % 64)) + } +} + func unionBitmapBitmap(a, b *Container) *Container { // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html @@ -2555,6 +2877,28 @@ func unionBitmapBitmap(a, b *Container) *Container { return output } +// unions bitmap b into bitmap a, mutating a in place. The n value of +// a will need to be repaired after the fact. +func unionBitmapBitmapInPlace(a, b *Container) { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + ) + + // Manually unroll loop to make it a little faster. + // TODO(rartoul): Can probably make this a few x faster using + // SIMD instructions. + for i := 0; i < bitmapN; i += 4 { + ab[i] |= bb[i] + ab[i+1] |= bb[i+1] + ab[i+2] |= bb[i+2] + ab[i+3] |= bb[i+3] + } +} + func difference(a, b *Container) *Container { if a.isArray() { if b.isArray() { @@ -3741,3 +4085,83 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } } } + +// handledIter and handledIters are wrappers around Bitmap Container iterators +// and assist with the unionIntoTarget algorithm by abstracting away some tedious +// operations. +type handledIter struct { + iter ContainerIterator + hasNext bool + handled bool +} + +type handledIters []handledIter + +func (w handledIters) next() bool { + hasNext := false + + for i, wrapped := range w { + next := wrapped.iter.Next() + w[i].hasNext = next + w[i].handled = false + if next { + hasNext = true + } + } + + return hasNext +} + +func (w handledIters) markItersWithCurrentKeyAsHandled(startIdx int, key uint64) { + for i := startIdx; i < len(w); i++ { + wrapped := w[i] + currKey, _ := wrapped.iter.Value() + if currKey == key { + w[i].handled = true + } + } +} + +func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummaryStats { + summary := containerUnionSummaryStats{} + + for _, iter := range w { + // Calculate key-level statistics here + currKey, currContainer := iter.iter.Value() + + if key == currKey { + summary.isOnlyContainerWithKey = false + summary.n += currContainer.n + + if currContainer.n == maxContainerVal+1 { + summary.hasMaxRange = true + summary.n = maxContainerVal + 1 + return summary + } + } + } + + return summary +} + +// Summary statistics about all the containers in the other bitmaps +// that share the same key so we can make smarter union strategy +// decisions. +type containerUnionSummaryStats struct { + // Estimated cardinality of the union of all containers with the same + // key across all bitmaps. This calculation is very rough as we just sum + // the cardinality of the container across the different bitmaps which could + // result in very inflated values, but it allows us to avoid allocating + // expensive bitmaps when unioning many low density containers. + n int32 + // Whether any other is the only container across all the bitmaps + // with the specified key. If true, we can skip all the unioning logic + // and just clone the container into target. + isOnlyContainerWithKey bool + // Whether any of the containers with the specified keys are storing every possible + // value that they can. If so, we can short-circuit all the unioning logic and use + // a RLE container with a single value in it. This is an optimization to + // avoid using an expensive bitmap container for bitmaps that have some + // extremely dense containers. + hasMaxRange bool +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 21c2b2088..993bcedb2 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3267,6 +3267,34 @@ func TestUnmarshalOfficialRoaring(t *testing.T) { } +func BenchmarkUnionBitmapBitmapInPlace(b *testing.B) { + b1 := newTestBitmapContainer() + b2 := newTestBitmapContainer() + for n := 0; n < b.N; n++ { + unionBitmapBitmapInPlace(b1, b2) + } +} + +func BenchmarkBitmapRepair(b *testing.B) { + b1 := newTestBitmapContainer() + for n := 0; n < b.N; n++ { + b1.bitmapRepair() + } +} + +func newTestBitmapContainer() *Container { + var ( + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + container = &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } + ) + return container +} + /* // This function exercises an arcane edge case in dead code. // It doesn't need to be run right now. diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f5c016ea3..4c3d15a4d 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -23,6 +23,7 @@ import ( "sort" "testing" "testing/quick" + "time" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" @@ -396,7 +397,132 @@ func TestBitmap_Union1(t *testing.T) { if n := result.Count(); n != 75007 { t.Fatalf("unexpected n: %d", n) } +} +func TestBitmap_UnionInPlace1(t *testing.T) { + var ( + bm0 = roaring.NewFileBitmap(0, 2683177) + bm1 = roaring.NewFileBitmap() + result = roaring.NewBitmap() + ) + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + bm1.Add(4000000) + + result.UnionInPlace(bm0, bm1) + if n := result.Count(); n != 2682675 { + t.Fatalf("unexpected n: %d", n) + } + + bm := testBM() + result = roaring.NewBitmap() + result.UnionInPlace(bm, bm0) + if n := result.Count(); n != 75009 { + t.Fatalf("unexpected n: %d", n) + } + + result = roaring.NewBitmap() + result.UnionInPlace(bm, bm) + if n := result.Count(); n != 75007 { + t.Fatalf("unexpected n: %d", n) + } + + // Make sure the bitmaps weren't mutated. + if n := bm0.Count(); n != 2 { + t.Fatalf("unexpected n: %d", n) + } + if n := bm1.Count(); n != 2682674 { + t.Fatalf("unexpected n: %d", n) + } +} + +// TestBitmap_UnionInPlaceProp is a manual property test that randomly generates +// a number of different bitmaps with random vals and unions them together. It +// then compares the result against a reference implementation (golang map) to +// ensure that all the unions were handled correctly. +func TestBitmap_UnionInPlaceProp(t *testing.T) { + var ( + seed = time.Now().UnixNano() + source = rand.NewSource(seed) + rng = rand.New(source) + numTests = 100 + maxNumIntsPerBatch = 100 + maxNumBatches = 100 + maxRangePercent = 2 + // Need to limit the range of possible numbers that we generate + // otherwise two randomly generated numbers landing in the same + // container would be extremely unlikely, leaving container merging + // behavior untested. + maxUint64Val = 1000000 + ) + + for i := 0; i < numTests; i++ { + var ( + // We will use sets as the "reference" implementation. + sets = []map[uint64]struct{}{} + bitmaps = []*roaring.Bitmap{} + ) + + // Ensure there are at least two batches. + numBatches := rng.Intn(maxNumBatches) + 2 + for j := 0; j < numBatches; j++ { + // For each "batch" create the equivalent set and bitmap. + var ( + set = map[uint64]struct{}{} + bitmap = roaring.NewBitmap() + ) + + if rng.Intn(100) <= maxRangePercent { + // Generate max range RLE containers with a configurable + // probability to ensure that code-path is exercised. + start := rng.Intn((maxUint64Val)) + // Add a continuous sequence of numbers that is 2x as long as the maximum + // size of a container to ensure we generate a maxRange container. + for x := start; x < (start + 2*(0xffff+1)); x++ { + set[uint64(x)] = struct{}{} + bitmap.Add(uint64(x)) + } + } + + // Generate and add a bunch of random values. + numIntsPerBatch := rng.Intn(maxNumIntsPerBatch) + for x := 0; x < numIntsPerBatch; x++ { + num := uint64(rng.Intn(maxUint64Val)) + set[num] = struct{}{} + bitmap.Add(num) + } + + sets = append(sets, set) + bitmaps = append(bitmaps, bitmap) + } + + // "Union" all the sets into the first one. + set0 := sets[0] + for _, set := range sets[1:] { + for val := range set { + set0[val] = struct{}{} + } + } + + // Union all the bitmaps into the first one. + bitmap0 := bitmaps[0] + bitmap0.UnionInPlace(bitmaps[1:]...) + + // Ensure the unioned set and bitmap have the same cardinality. + if len(set0) != int(bitmap0.Count()) { + t.Fatalf("cardinality of set is: %d, but bitmap is: %d, failed with seed: %d", + len(set0), bitmap0.Count(), seed) + } + + // Ensure the unioned set and bitmap have the exact same values. + for val := range set0 { + if !bitmap0.Contains(val) { + t.Fatalf("set contained %d, but bitmap did not, failed with seed: %d", + val, seed) + } + } + } } func TestBitmap_Intersection_Empty(t *testing.T) { @@ -544,11 +670,35 @@ func TestBitmap_Union(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) result := bm0.Union(bm1) + if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) } } +func TestBitmap_UnionInPlace(t *testing.T) { + var ( + bm0 = roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 = roaring.NewFileBitmap(0, 50000, 1000001, 1000002) + result = roaring.NewBitmap() + ) + result.UnionInPlace(bm0, bm1) + + // Make sure the union worked. + if n := result.Count(); n != 5 { + t.Fatalf("unexpected n: %d", n) + } + + // Make sure the other bitmaps weren't mutated. + if n := bm0.Count(); n != 4 { + t.Fatalf("unexpected n: %d", n) + } + if n := bm1.Count(); n != 4 { + t.Fatalf("unexpected n: %d", n) + } + +} + func TestBitmap_Xor(t *testing.T) { bm0 := testBM() bm1 := roaring.NewFileBitmap(0, 1, 2, 3) @@ -1350,3 +1500,23 @@ func BenchmarkSliceDescending(b *testing.B) { } } } + +func BenchmarkUnion(b *testing.B) { + data := getBenchData(b) + for n := 0; n < b.N; n++ { + data.a1. + Union(data.a2). + Union(data.b). + Union(data.r1). + Union(data.r2) + } +} + +func BenchmarkUnionBulk(b *testing.B) { + data := getBenchData(b) + bm := roaring.NewBitmap() + for n := 0; n < b.N; n++ { + bm. + UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2) + } +} diff --git a/row.go b/row.go index 6a5f541c4..2a0eddf58 100644 --- a/row.go +++ b/row.go @@ -42,6 +42,19 @@ func NewRow(columns ...uint64) *Row { return r } +func (r *Row) IsEmpty() bool { + if len(r.segments) == 0 { + return true + } + for i := range r.segments { + if r.segments[i].n > 0 { + return false + } + + } + return true +} + // Merge merges data from other into r. func (r *Row) Merge(other *Row) { var segments []rowSegment diff --git a/row_test.go b/row_test.go index 7f1279ceb..49559f131 100644 --- a/row_test.go +++ b/row_test.go @@ -111,3 +111,17 @@ func TestRow_Difference_Segment(t *testing.T) { t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp) } } + +func TestRow_IsEmpty(t *testing.T) { + r1 := pilosa.NewRow(1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + res := r2.Intersect(r1) + + if r1.IsEmpty() { + t.Fatal("r1 Should Not Be Empty\n") + } + if !res.IsEmpty() { + t.Fatal("Result Should Be Empty\n") + } + +} diff --git a/server.go b/server.go index 2e1e7091f..ca5bfcce4 100644 --- a/server.go +++ b/server.go @@ -428,7 +428,7 @@ func (s *Server) SyncData() error { } func (s *Server) monitorAntiEntropy() { - if s.antiEntropyInterval == 0 { + if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 { return // anti entropy disabled } s.cluster.initializeAntiEntropy() diff --git a/server/cluster_test.go b/server/cluster_test.go index 9227a2b53..90467f4b2 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -206,6 +206,16 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,1300000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + // Configure node1 m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" @@ -221,6 +231,18 @@ func TestClusterResize_AddNode(t *testing.T) { } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } }) t.Run("SkippedShard", func(t *testing.T) { // Configure node0 @@ -247,6 +269,16 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,2400000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + // Configure node1 m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" @@ -262,6 +294,18 @@ func TestClusterResize_AddNode(t *testing.T) { } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } }) } diff --git a/server/handler_test.go b/server/handler_test.go index 5ae4dd877..f6652632d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -713,6 +713,91 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected body: %q", w.Body.String()) } }) + + t.Run("translate keys", func(t *testing.T) { + // create index + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i1-tr", strings.NewReader(`{"options":{"keys":true}}`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // create field + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/i1-tr/field/f1", strings.NewReader(`{"options":{"keys":true}}`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // set some bits + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/i1-tr/query", strings.NewReader(`Set("col1", f1="row1")`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + // Generate request body for translate column keys request + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ + Index: "i1-tr", + Keys: []string{"col1", "col2", "col3"}, + }) + if err != nil { + t.Fatal(err) + } + // Generate protobuf request. + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + target := []uint64{1, 2, 3} + resp := pilosa.TranslateKeysResponse{} + err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, resp.IDs) { + t.Fatalf("%v != %v", target, resp.IDs) + } + + // Generate request body for translate row keys request + reqBody, err = cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ + Index: "i1-tr", + Field: "f1", + Keys: []string{"row1", "row2"}, + }) + if err != nil { + t.Fatal(err) + } + // Generate protobuf request. + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + target = []uint64{1, 2} + resp = pilosa.TranslateKeysResponse{} + err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, resp.IDs) { + t.Fatalf("%v != %v", target, resp.IDs) + } + }) } func TestClusterTranslator(t *testing.T) { diff --git a/server/server.go b/server/server.go index c6b69a7f5..c80f8268e 100644 --- a/server/server.go +++ b/server/server.go @@ -174,30 +174,6 @@ func (m *Command) Wait() error { } } -// setupLogger sets up the logger based on the configuration. -func (m *Command) setupLogger() error { - if m.Config.LogPath == "" { - m.logOutput = m.Stderr - } else { - f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return errors.Wrap(err, "opening file") - } - m.logOutput = f - err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) - if err != nil { - return errors.Wrap(err, "dup2ing stderr onto logfile") - } - } - - if m.Config.Verbose { - m.logger = logger.NewVerboseLogger(m.logOutput) - } else { - m.logger = logger.NewStandardLogger(m.logOutput) - } - return nil -} - // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { err := m.setupLogger() diff --git a/server/server_test.go b/server/server_test.go index 3aa11b8a2..a2fb5b5d7 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -15,8 +15,10 @@ package server_test import ( + "bytes" "context" "encoding/json" + "flag" "fmt" "io/ioutil" "math/rand" @@ -31,10 +33,18 @@ import ( "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" + "golang.org/x/sync/errgroup" ) +var runStress bool + +func init() { // nolint: gochecknoinits + flag.BoolVar(&runStress, "stress", false, "Enable stress tests (time consuming)") +} + // Ensure program can process queries and maintain consistency. func TestMain_Set_Quick(t *testing.T) { if testing.Short() { @@ -237,6 +247,59 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } +func TestMain_GroupBy(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + // Create fields. + client := m.Client() + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "generalk", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "subk", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + query := ` + Set(0, generalk="ten") + Set(1, generalk="ten") + Set(1001, generalk="ten") + Set(2, generalk="eleven") + Set(1002, generalk="eleven") + Set(2, generalk="twelve") + Set(1002, generalk="twelve") + + Set(0, subk="one-hundred") + Set(1, subk="one-hundred") + Set(3, subk="one-hundred") + Set(1001, subk="one-hundred") + Set(2, subk="one-hundred-ten") + Set(0, subk="one-hundred-ten") + ` + + // Set columns on row. + if _, err := m.Query("i", "", query); err != nil { + t.Fatal(err) + } + + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "eleven"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "twelve"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + } + + // Query row. + if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil { + t.Fatal(err) + } else { + test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) + } +} + // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { if c, err := ParseConfig(`bind = "local"`); err != nil { @@ -678,6 +741,15 @@ func TestClusterQueriesAfterRestart(t *testing.T) { defer cluster.Close() cmd1 := cluster[1] + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) @@ -745,3 +817,97 @@ func TestClusterQueriesAfterRestart(t *testing.T) { } // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address. + +func TestClusterExhaustingConnections(t *testing.T) { + if !runStress { + t.Skip("stress") + } + cluster := test.MustRunCluster(t, 5) + defer cluster.Close() + cmd1 := cluster[1] + + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + eg := errgroup.Group{} + for i := 0; i < 20; i++ { + i := i + eg.Go(func() error { + for j := i; j < 10000; j += 20 { + _, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth), + }) + if err != nil { + return err + } + } + return nil + }) + } + err := eg.Wait() + if err != nil { + t.Fatalf("setting lots of shards: %v", err) + } +} + +func TestClusterExhaustingConnectionsImport(t *testing.T) { + if !runStress { + t.Skip("stress") + } + cluster := test.MustRunCluster(t, 5) + defer cluster.Close() + cmd1 := cluster[1] + + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + bm := roaring.NewBitmap() + bm.DirectAdd(0) + buf := &bytes.Buffer{} + bm.WriteTo(buf) + data := buf.Bytes() + + eg := errgroup.Group{} + for i := uint64(0); i < 20; i++ { + i := i + eg.Go(func() error { + for j := i; j < 10000; j += 20 { + if (j-i)%1000 == 0 { + fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000) + } + err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{ + Views: map[string][]byte{ + "": data, + }, + }) + if err != nil { + return err + } + } + return nil + }) + } + err := eg.Wait() + if err != nil { + t.Fatalf("setting lots of shards: %v", err) + } +} diff --git a/server/setup_logger.go b/server/setup_logger.go new file mode 100644 index 000000000..06d61eced --- /dev/null +++ b/server/setup_logger.go @@ -0,0 +1,35 @@ +// +build !arm64 + +package server + +import ( + "os" + "syscall" + + "github.com/pilosa/pilosa/logger" + "github.com/pkg/errors" +) + +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrap(err, "opening file") + } + m.logOutput = f + err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) + if err != nil { + return errors.Wrap(err, "dup2ing stderr onto logfile") + } + } + + if m.Config.Verbose { + m.logger = logger.NewVerboseLogger(m.logOutput) + } else { + m.logger = logger.NewStandardLogger(m.logOutput) + } + return nil +} diff --git a/server/setup_logger_arm64.go b/server/setup_logger_arm64.go new file mode 100644 index 000000000..dd6718741 --- /dev/null +++ b/server/setup_logger_arm64.go @@ -0,0 +1,33 @@ +package server + +import ( + "os" + "syscall" + + "github.com/pilosa/pilosa/logger" + "github.com/pkg/errors" +) + +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrap(err, "opening file") + } + m.logOutput = f + err = syscall.Dup3(int(f.Fd()), int(os.Stderr.Fd()), 0) + if err != nil { + return errors.Wrap(err, "dup2ing stderr onto logfile") + } + } + + if m.Config.Verbose { + m.logger = logger.NewVerboseLogger(m.logOutput) + } else { + m.logger = logger.NewStandardLogger(m.logOutput) + } + return nil +} diff --git a/test/pilosa.go b/test/pilosa.go index 3bc43d593..31e93bb1d 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -22,12 +22,14 @@ import ( gohttp "net/http" "os" "path" + "reflect" "strconv" "strings" "testing" "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" @@ -69,7 +71,7 @@ func newCommand(opts ...server.CommandOption) *Command { m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true - m.Config.Translation.MapSize = 100000 + m.Config.Translation.MapSize = 140000 if testing.Verbose() { m.Command.Stdout = os.Stdout @@ -185,6 +187,49 @@ func (m *Command) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +func (m *Command) QueryProtobuf(indexName string, query string) (*pilosa.QueryResponse, error) { + var ser proto.Serializer + queryReq := &pilosa.QueryRequest{ + Index: indexName, + Query: query, + } + body, err := ser.Marshal(queryReq) + if err != nil { + return nil, err + } + + req, err := gohttp.NewRequest( + "POST", + fmt.Sprintf("%s/index/%s/query", m.URL(), indexName), + bytes.NewReader(body), + ) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + + resp, err := gohttp.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + response := &pilosa.QueryResponse{} + err = ser.Unmarshal(buf, response) + if err != nil { + return nil, err + } + + return response, nil +} + // RecalculateCaches is deprecated. Use MustRecalculateCaches. func (m *Command) RecalculateCaches() error { resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") @@ -380,6 +425,17 @@ func MustDo(method, urlStr string, body string) *httpResponse { return &httpResponse{Response: resp, Body: string(buf)} } +func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { + if len(results) != len(expected) { + t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) + } + for i, result := range results { + if !reflect.DeepEqual(expected[i], result) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } +} + // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { *gohttp.Response diff --git a/translate.go b/translate.go index 2c3125ad9..b86041e3d 100644 --- a/translate.go +++ b/translate.go @@ -29,10 +29,11 @@ const ( ) var ( - ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") - ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") - ErrReplicationNotSupported = errors.New("pilosa: replication not supported") - ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized") ) // TranslateStore is the storage for translation string-to-uint64 values. @@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) { return 0, nil } - // Shorten buffer to maximum read size. - if max := sz - r.offset; int64(len(p)) > max { + if max := sz - r.offset; max > int64(len(p)) { + // If p is not large enough to hold a single entry, + // return an error so the client can increase the + // size of p and try again. + return 0, ErrTranslateReadTargetUndersized + } else if int64(len(p)) > max { + // Shorten buffer to maximum read size. p = p[:max] } diff --git a/utils_internal_test.go b/utils_internal_test.go index ff149e8cf..21d8d0780 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -371,10 +372,26 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. - if err := destCluster.holder.applySchema(instr.Schema); err != nil { + if err := destCluster.holder.applySchema(instr.NodeStatus.Schema); err != nil { return err } + // Sync available shards. + for _, is := range instr.NodeStatus.Indexes { + for _, fs := range is.Fields { + f := destCluster.holder.Field(is.Name, fs.Name) + + // if we don't know about a field locally, log an error because + // fields should be created and synced prior to shard creation + if f == nil { + continue + } + if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } + } + } + for _, src := range instr.Sources { srcCluster := t.clusterByID(src.Node.ID) diff --git a/view.go b/view.go index 128e3b828..00dcf02c6 100644 --- a/view.go +++ b/view.go @@ -206,37 +206,45 @@ func (v *view) recalculateCaches() { // CreateFragmentIfNotExists returns a fragment in the view by shard. func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { - v.mu.Lock() - defer v.mu.Unlock() - return v.createFragmentIfNotExists(shard) + frag, msg, err := v.createFragmentIfNotExists(shard) + + // if msg is not nil, then a new shard was created + if err == nil && msg != nil { + // Broadcast a message that a new max shard was just created. + if err = v.broadcaster.SendSync(msg); err != nil { + frag.close() + return nil, errors.Wrap(err, "sending createshard message") + } + v.mu.Lock() + v.fragments[shard] = frag + v.mu.Unlock() + } + + return frag, err } -func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { +func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardMessage, error) { + v.mu.Lock() + defer v.mu.Unlock() // Find fragment in cache first. if frag := v.fragments[shard]; frag != nil { - return frag, nil + return frag, nil, nil } // Initialize and open fragment. frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { - return nil, errors.Wrap(err, "opening fragment") + return nil, nil, errors.Wrap(err, "opening fragment") } frag.RowAttrStore = v.rowAttrStore - // Broadcast a message that a new max shard was just created. - if err := v.broadcaster.SendSync(&CreateShardMessage{ + msg := &CreateShardMessage{ Index: v.index, Field: v.field, Shard: shard, - }); err != nil { - frag.close() - return nil, errors.Wrap(err, "sending createshard message") } - // Save to lookup. - v.fragments[shard] = frag - return frag, nil + return frag, msg, nil } func (v *view) newFragment(path string, shard uint64) *fragment {