diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9abb766ce..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 2ddff4f4b..3a9abe355 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,17 @@ language: go go: - - 1.8 - 1.9 + - "1.10" # Use string, as 1.10==1.1 if interpreted as float. - master env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" + matrix: + - GOARCH=386 + - GOARCH=amd64 install: - - make vendor + - make install-dep install-statik vendor generate-statik script: - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 @@ -19,10 +22,12 @@ before_deploy: - pip install awscli --user `whoami` deploy: - provider: script - script: make prerelease-upload + script: make prerelease-upload GOOS=linux GOARCH=amd64 skip_cleanup: true on: - branch: master + all_branches: true + go: "1.10" + condition: $GOARCH = amd64 matrix: allow_failures: - go: master diff --git a/CHANGELOG.md b/CHANGELOG.md index 9676b4f89..5ab4ad1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,135 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v0.9.0] - 2018-05-04 + +This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions. + +*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* + +### Added + +- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) +- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) +- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) +- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) +- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) +- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) +- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) +- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) +- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) +- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) +- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) +- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) +- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) +- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) +- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) +- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) + +### Changed + +- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) +- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) +- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) +- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) +- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) +- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) +- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) +- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) +- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) +- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) +- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) +- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) +- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) +- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) +- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) +- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) +- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) +- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) + +### Removed + +- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) +- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) +- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) +- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) +- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) +- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) +- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) +- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) +- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) + +### Fixed + +- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) +- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) +- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) +- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) +- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) +- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) +- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) +- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) +- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) +- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) +- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) +- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) + +### Deprecated + +- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) + +### Performance + +- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) + +## [0.8.8] - 2018-02-19 + +This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions. + +### Fixed + +- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) + +## [0.8.7] - 2018-02-12 + +This version contains 1 contribution from 1 contributors. There are 2 files changed, 84 insertions, and 4 deletions. + +### Fixed + +- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) + +## [0.8.6] - 2018-02-09 + +This version contains 2 contributions from 2 contributors. There are 3 files changed, 171 insertions, and 6 deletions. + +### Fixed + +- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) +- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) + +## [0.8.5] - 2018-01-18 + +This version contains 1 contribution from 1 contributor. There is 1 file changed, 1 insertion, and 0 deletions. + +### Fixed + +- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) + +## [0.8.4] - 2018-01-10 + +This version contains 4 contributions from 3 contributors. There are 17 files changed, 974 insertions, and 221 deletions. + +### Fixed + +- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) +- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) + +### Performance + +- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) +- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) + ## [0.8.3] - 2017-12-12 This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d15c42ed7..925f789b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,30 +2,155 @@ ## Reporting a bug -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1] +If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. ## Submitting a feature request -Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4] +Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] which starts by [opening a new issue][1] that describes the new feature proposal. -## Submitting code changes +## Making code contributions Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing before you start working, otherwise you run the risk of duplicating effort. This also +you're doing, otherwise you run the risk of duplicating effort. This also gives others an opportunity to provide input for your feature. If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. -- Fork the [Pilosa repository][2] and then clone your fork: - ```shell - git clone git@github.com:/pilosa.git +### Development Environment + +- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information. + +- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). + +- Fork the [Pilosa repository][2] to your own account. + +- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: + + ```sh + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone git@github.com:${USER}/pilosa.git + ``` + +- `cd` to your pilosa directory: + + ```sh + cd ${GOPATH}/src/github.com/pilosa/pilosa + ``` + +- [Install](https://github.com/golang/dep/#installation) `dep` to manage dependencies: + + ```sh + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + ``` + +- Install Pilosa command line tools: + + ```sh + make install + ``` + + or + + ``` + dep ensure && go install github.com/pilosa/pilosa/cmd/... + ``` + + Running `pilosa` should now run a Pilosa instance. + +- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: + + ```sh + cd ${GOPATH}/src/github.com/pilosa/pilosa + git remote add upstream git@github.com:pilosa/pilosa.git + ``` + +### Makefile + +Pilosa includes a Makefile that automates several tasks: + +- Install Pilosa: + + ```sh + make install + ``` + +- Install build dependencies (dep, statik, and protoc): + + ```sh + make install-build-deps + ``` + +- Create the vendor directory: + + ```sh + make vendor + ``` + +- Run the test suite: + + ```sh + make test + ``` + +- View the coverage report: + + ```sh + make cover-viz + ``` + +- Clear the `vendor/` and `build/` directories: + + ```sh + make clean + ``` + +- Create release tarballs: + + ```sh + make release + ``` + +- Generate static assets for the WebUI: + + ```sh + make generate-statik + ``` + +- Regenerate protocol buffer files in `internal/`: + + ```sh + make generate-protoc + ``` + +- Create tagged Docker image: + + ```sh + make docker + ``` + +- Run tests inside Docker container: + + ```sh + make docker-test + ``` + +Additional commands are available in the `Makefile`. + +### Submitting code changes + +- Before starting to work on a task, sync your branch with the upstream: + + ```sh + git fetch upstream + git checkout master + git merge upstream/master ``` - Create a local feature branch: - ```shell + ```sh git checkout -b something-amazing ``` @@ -33,13 +158,13 @@ If you want to help but you aren't sure where to start, check out our [github la - Make sure that you've written tests for your new feature, and then run the tests: - ```shell + ```sh make test ``` - Verify that your pull request is applied to the latest version of code on github: - ```shell + ```sh git remote add upstream git@github.com:pilosa/pilosa.git git fetch upstream git rebase -i upstream/master @@ -47,7 +172,7 @@ If you want to help but you aren't sure where to start, check out our [github la - Push to your fork: - ```shell + ```sh git push -u something-amazing ``` @@ -59,4 +184,4 @@ If you want to help but you aren't sure where to start, check out our [github la [3]: https://github.com/pilosa/pilosa/compare/ [4]: https://github.com/pilosa/general/blob/master/proposal.md [5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer \ No newline at end of file +[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/Dockerfile b/Dockerfile index fb3d5cc94..7cdb93cc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,9 @@ -FROM golang:1.9.2 as builder +FROM golang:1.10.2 as builder -ARG ldflags='' - -COPY . /go/src/github.com/pilosa/pilosa +COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && make vendor \ - && CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa + && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" FROM scratch diff --git a/Gopkg.lock b/Gopkg.lock index 7837cadff..f29660b22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,12 @@ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" +[[projects]] + name = "github.com/cespare/xxhash" + packages = ["."] + revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" + version = "v1.0.0" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -51,7 +57,10 @@ [[projects]] name = "github.com/go-ole/go-ole" - packages = [".","oleutil"] + packages = [ + ".", + "oleutil" + ] revision = "0e87ea779d9deb219633b828a023b32e1244dd57" version = "v1.2.0" @@ -61,12 +70,6 @@ revision = "100ba4e885062801d56799d78530b73b178a78f3" version = "v0.4" -[[projects]] - branch = "master" - name = "github.com/golang/groupcache" - packages = ["lru"] - revision = "84a468cf14b4376def5d68c722b139b881c450a4" - [[projects]] branch = "master" name = "github.com/golang/protobuf" @@ -82,8 +85,8 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "7f08801859139f86dfafd1c296e2cba9a80d292e" - version = "v1.6.0" + revision = "53c1911da2b537f792e7cafcb446b05ffe33b996" + version = "v1.6.1" [[projects]] branch = "master" @@ -124,7 +127,17 @@ [[projects]] branch = "master" name = "github.com/hashicorp/hcl" - packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] @@ -148,7 +161,10 @@ [[projects]] branch = "master" name = "github.com/miekg/dns" - packages = [".","internal/socket"] + packages = [ + ".", + "internal/socket" + ] revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" [[projects]] @@ -189,20 +205,30 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["host","internal/common","mem","process"] + packages = [ + "cpu", + "host", + "internal/common", + "mem", + "net", + "process" + ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" [[projects]] - name = "github.com/sony/gobreaker" + branch = "master" + name = "github.com/shirou/w32" packages = ["."] - revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36" - version = "0.3.0" + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" [[projects]] branch = "master" name = "github.com/spf13/afero" - packages = [".","mem"] + packages = [ + ".", + "mem" + ] revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" [[projects]] @@ -250,13 +276,23 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = [ + "unix", + "windows" + ] revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm" + ] revision = "88f656faf3f37f690df1a32515b479415e1a6769" [[projects]] @@ -268,6 +304,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" + inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index a76225bd8..e4d120659 100644 --- a/Makefile +++ b/Makefile @@ -1,120 +1,138 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test -DEP := $(shell command -v dep 2>/dev/null) -STATIK := $(shell command -v statik 2>/dev/null) -PROTOC := $(shell command -v protoc 2>/dev/null) -VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -STATUS := $(shell git status --porcelain) -IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa -PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) -BUILD_TIME=`date -u +%FT%T%z` +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" -DOCKER_GOLANG_IMAGE=golang:latest +GO_VERSION=latest -default: test pilosa +# Run tests and compile Pilosa +default: test build +# Remove vendor and build directories clean: rm -rf vendor build -$(GOPATH)/bin: - mkdir $(GOPATH)/bin - -dep: $(GOPATH)/bin - go get -u github.com/golang/dep/cmd/dep - +# Set up vendor directory using `dep` vendor: Gopkg.toml -ifndef DEP - make dep -endif + $(MAKE) require-dep dep ensure touch vendor -Gopkg.lock: dep Gopkg.toml - dep ensure - +# Run test suite test: vendor - go test $(PKGS) $(TESTFLAGS) + go test ./... $(TESTFLAGS) +# Run test suite with coverage enabled cover: vendor - mkdir -p build/coverage - echo "mode: set" > build/coverage/all.out - for pkg in $(PKGS) ; do \ - make cover-pkg PKG=$$pkg ; \ - done - -cover-pkg: - mkdir -p build/coverage - touch build/coverage/$(subst /,-,$(PKG)).out - go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) - tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + mkdir -p build + $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" +# Run test suite with coverage enabled and view coverage results in browser cover-viz: cover - go tool cover -html=build/coverage/all.out + go tool cover -html=build/coverage.out -pilosa: vendor - go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Compile Pilosa +build: vendor + go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release build under the build directory release-build: vendor -ifdef DOCKER_BUILD - make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -else - make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -endif - cp NOTICE LICENSE README.md build/pilosa-$(IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ - @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz -release: -ifeq ($(STATUS),"") - make release-build GOOS=darwin GOARCH=amd64 - make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 - make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 -else - @echo "Will not create release with unclean git status." -endif +# Error out if there are untracked changes in Git +check-clean: + $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) +# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. +release: check-clean + $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 + +# Create branch-tagged pre-release for client library CI jobs prerelease-build: vendor - make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" - cp NOTICE LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH) - tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/ - @echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz" + $(MAKE) release-build VERSION_ID=$(BRANCH_ID) +# Create prerelease build for Linux/amd64 prerelease: - make prerelease-build GOOS=linux GOARCH=amd64 + $(MAKE) prerelease-build GOOS=linux GOARCH=amd64 +# Upload prerelease to S3 prerelease-upload: prerelease - aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read + aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read +# Install Pilosa install: vendor - go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -.protoc-gen-gofast: vendor -ifndef PROTOC - $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases") -endif - go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast - cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast - -generate-protoc: .protoc-gen-gofast +# `go generate` protocol buffers +generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -generate-statik: statik +# `go generate` statik assets (WebUI) +generate-statik: require-statik + go generate github.com/pilosa/pilosa/statik + +# `go generate` stringers +generate-stringer: go generate github.com/pilosa/pilosa -generate: generate-protoc generate-statik - -statik: -ifndef STATIK - go get github.com/rakyll/statik -endif +# `go generate` all needed packages +generate: generate-protoc generate-statik generate-stringer +# Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . - @echo "Created image: pilosa:$(VERSION)" + docker build -t "pilosa:$(VERSION)" . + @echo Created docker image: pilosa:$(VERSION) +# Compile Pilosa inside Docker container docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./... + +###################### +# Build dependencies # +###################### + +# Verifies that needed build dependency is installed. Errors out if not installed. +define require + $(if $(shell command -v $1 2>/dev/null), + $(info Verified build dependency "$1" is installed.), + $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`)) +endef + +require-dep: + $(call require,dep) + +require-statik: + $(call require,statik) + +require-protoc-gen-gofast: + $(call require,protoc-gen-gofast) + +require-protoc: + $(call require,protoc) + +install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc + +install-dep: + go get -u github.com/golang/dep/cmd/dep + +install-statik: + go get -u github.com/rakyll/statik + +install-protoc-gen-gofast: + go get -u github.com/gogo/protobuf/protoc-gen-gofast + +install-protoc: + @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html diff --git a/NOTICE b/NOTICE index b93e9a17d..dc9506565 100644 --- a/NOTICE +++ b/NOTICE @@ -18,6 +18,23 @@ limitations under the License. Third-party software licenses ============================= +The file /pilosa/lru/lru.go contains a redistribution of lru +(github.com/golang/groupcache/lru); the license follows: + + Copyright 2013 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + The file /pilosa/roaring/btree.go contains a modified redistribution of b (https://github.com/cznic/b); the license follows: diff --git a/README-dev.md b/README-dev.md deleted file mode 100644 index 809befbc3..000000000 --- a/README-dev.md +++ /dev/null @@ -1,69 +0,0 @@ - -Development Environment -======================= - -Install Go versions 1.6.2+ or 1.7 for your platform. - -Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private. - -Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`. - -Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo: - -```sh -mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ -git clone git@github.com:${USER}/pilosa.git -``` - -`cd` to your pilosa directory: - -```sh -cd ${GOPATH}/src/github.com/pilosa/pilosa -``` - -Install `dep` to manage dependencies: - -```sh -go get -u github.com/golang/dep/cmd/dep -``` - -Install Pilosa command line tools: - -```sh -make install -# or: -# dep ensure && go install github.com/pilosa/pilosa/cmd/... -``` - -Running `pilosa` should now run a Pilosa instance. - -In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo: - -```sh -cd ${GOPATH}/src/github.com/pilosa/pilosa -git remote add upstream git@github.com:pilosa/pilosa.git -``` - -Before starting to work on a task, sync your branch with the upstream: - -```sh -git fetch upstream -git checkout master -git merge upstream/master -``` - -Create a branch for the task: - -```sh -git checkout -b a-branch-for-the-task -``` - -Update the code in the branch, and commit it. - -Push it to your own repo: - -```sh -git push --set-upstream origin a-branch-for-the-task -``` - -All left to do is creating a pull request on github.com. diff --git a/api.go b/api.go new file mode 100644 index 000000000..691e076ff --- /dev/null +++ b/api.go @@ -0,0 +1,1215 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate stringer -type=apiMethod + +package pilosa + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" +) + +// API provides the top level programmatic interface to Pilosa. It is usually +// wrapped by a handler which provides an external interface (e.g. HTTP). +type API struct { + Holder *Holder + // The execution engine for running queries. + Executor interface { + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + } + Broadcaster Broadcaster + BroadcastHandler BroadcastHandler + StatusHandler StatusHandler + Cluster *Cluster + RemoteClient *http.Client + Logger Logger +} + +// NewAPI returns a new API instance. +func NewAPI() *API { + return &API{ + Broadcaster: NopBroadcaster, + //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop + //StatusHandler: NopStatusHandler, // TODO: implement the nop + Logger: NopLogger, + } +} + +// validAPIMethods specifies the api methods that are valid for each +// cluster state. +var validAPIMethods = map[string]map[apiMethod]struct{}{ + ClusterStateStarting: methodsCommon, + ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + ClusterStateResizing: appendMap(methodsCommon, methodsResizing), +} + +func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { + r := make(map[apiMethod]struct{}) + for k, v := range a { + r[k] = v + } + for k, v := range b { + r[k] = v + } + return r +} + +func (api *API) validate(f apiMethod) error { + state := api.Cluster.State() + if _, ok := validAPIMethods[state][f]; ok { + return nil + } + return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)} +} + +// Query parses a PQL query out of the request and executes it. +func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + if err := api.validate(apiQuery); err != nil { + return QueryResponse{}, errors.Wrap(err, "validate api method") + } + + resp := QueryResponse{} + + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + return resp, err + } + execOpts := &ExecOptions{ + Remote: req.Remote, + ExcludeAttrs: req.ExcludeAttrs, + ExcludeBits: req.ExcludeBits, + } + results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + if err != nil { + return resp, err + } + resp.Results = results + + // Fill column attributes if requested. + if req.ColumnAttrs && !req.ExcludeBits { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Bitmap) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := api.readColumnAttrSets(api.Holder.Index(req.Index), columnIDs) + if err != nil { + return resp, err + } + resp.ColumnAttrSets = columnAttrSets + } + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, err + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil +} + +// CreateIndex makes a new Pilosa index. +func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + if err := api.validate(apiCreateIndex); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Create index. + index, err := api.Holder.CreateIndex(indexName, options) + if err != nil { + return nil, err + } + // Send the create index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateIndexMessage{ + Index: indexName, + Meta: options.Encode(), + }) + if err != nil { + api.Logger.Printf("problem sending CreateIndex message: %s", err) + return nil, err + } + api.Holder.Stats.Count("createIndex", 1, 1.0) + return index, nil +} + +// Index retrieves the named index. +func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { + if err := api.validate(apiIndex); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + return index, nil +} + +// DeleteIndex removes the named index. If the index is not found it does +// nothing and returns no error. +func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + if err := api.validate(apiDeleteIndex); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Delete index from the holder. + err := api.Holder.DeleteIndex(indexName) + if err != nil { + return err + } + // Send the delete index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.DeleteIndexMessage{ + Index: indexName, + }) + if err != nil { + api.Logger.Printf("problem sending DeleteIndex message: %s", err) + return err + } + api.Holder.Stats.Count("deleteIndex", 1, 1.0) + return nil +} + +// CreateFrame makes the named frame in the named index with the given options. +func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + if err := api.validate(apiCreateFrame); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Create frame. + frame, err := index.CreateFrame(frameName, options) + if err != nil { + return nil, err + } + + // Send the create frame message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateFrameMessage{ + Index: indexName, + Frame: frameName, + Meta: options.Encode(), + }) + if err != nil { + api.Logger.Printf("problem sending CreateFrame message: %s", err) + return nil, err + } + api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return frame, nil +} + +// DeleteFrame removes the named frame from the named index. If the index is not +// found, an error is returned. If the frame is not found, it is ignored and no +// action is taken. +func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + if err := api.validate(apiDeleteFrame); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete frame from the index. + if err := index.DeleteFrame(frameName); err != nil { + return err + } + + // Send the delete frame message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFrameMessage{ + Index: indexName, + Frame: frameName, + }) + if err != nil { + api.Logger.Printf("problem sending DeleteFrame message: %s", err) + return err + } + api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return nil +} + +// ExportCSV encodes the fragment designated by the index,frame,view,slice as +// CSV of the form , +func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + if err := api.validate(apiExportCSV); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Validate that this handler owns the slice. + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) + return ErrClusterDoesNotOwnSlice + } + + // Find the fragment. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return ErrFragmentNotFound + } + + // Wrap writer with a CSV writer. + cw := csv.NewWriter(w) + + // Iterate over each bit. + if err := f.ForEachBit(func(rowID, columnID uint64) error { + return cw.Write([]string{ + strconv.FormatUint(rowID, 10), + strconv.FormatUint(columnID, 10), + }) + }); err != nil { + return err + } + + // Ensure data is flushed. + cw.Flush() + + return nil +} + +// SliceNodes returns the node and all replicas which should contain a slice's data. +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { + if err := api.validate(apiSliceNodes); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + return api.Cluster.SliceNodes(indexName, slice), nil +} + +// MarshalFragment returns an object which can write the specified fragment's data +// to an io.Writer. The serialized data can be read back into a fragment with +// the UnmarshalFragment API call. +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { + if err := api.validate(apiMarshalFragment); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + return f, nil +} + +// UnmarshalFragment creates a new fragment (if necessary) and reads data from a +// Reader which was previously written by MarshalFragment to populate the +// fragment's data. +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + if err := api.validate(apiUnmarshalFragment); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve view. + view, err := f.CreateViewIfNotExists(viewName) + if err != nil { + return err + } + + // Retrieve fragment from frame. + frag, err := view.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Read fragment in from request body. + if _, err := frag.ReadFrom(reader); err != nil { + return err + } + return nil +} + +// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to +// return anything useful. Currently it returns protobuf encoded row and column +// ids from a "block" which is a subdivision of a fragment. +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + if err := api.validate(apiFragmentBlockData); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + reqBytes, err := ioutil.ReadAll(body) + if err != nil { + return nil, BadRequestError{errors.Wrap(err, "read body error")} + } + var req internal.BlockDataRequest + if err := proto.Unmarshal(reqBytes, &req); err != nil { + return nil, BadRequestError{errors.Wrap(err, "unmarshal body error")} + } + + // Retrieve fragment from holder. + f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) + if f == nil { + return nil, ErrFragmentNotFound + } + + var resp = internal.BlockDataResponse{} + resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + + // Encode response. + buf, err := proto.Marshal(&resp) + if err != nil { + return nil, errors.Wrap(err, "merge block response encoding error") + } + + return buf, nil +} + +// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. +func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + if err := api.validate(apiFragmentBlocks); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + + // Retrieve blocks. + blocks := f.Blocks() + return blocks, nil +} + +// RestoreFrame reads all the data that this host should have for a given frame +// from replicas in the cluster and restores that data to it. +func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + if err := api.validate(apiRestoreFrame); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Create a client for the remote cluster. + client := NewInternalHTTPClientFromURI(host, api.RemoteClient) + + // Determine the maximum number of slices. + maxSlices, err := client.MaxSliceByIndex(ctx) + if err != nil { + return err + } + + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve list of all views. + views, err := client.FrameViews(ctx, indexName, frameName) + if err != nil { + return err + } + + // Loop over each slice and import it if this node owns it. + for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { + // Ignore this slice if we don't own it. + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + continue + } + + // Loop over view names. + for _, view := range views { + // Create view. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + return err + } + + // Otherwise retrieve the local fragment. + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Stream backup from remote node. + rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice) + if err != nil { + return err + } else if rd == nil { + continue // slice doesn't exist + } + + // Restore to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + return err + } + } + } + + return nil +} + +// Hosts returns a list of the hosts in the cluster including their ID, +// URL, and which is the coordinator. +func (api *API) Hosts(ctx context.Context) []*Node { + return api.Cluster.Nodes +} + +// CreateInputDefinition is deprecated and will be removed. Do not use it. +func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + if err := api.validate(apiCreateInputDefinition); err != nil { + return errors.Wrap(err, "validate api method") + } + + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. +Please open an issue if you need to continue using it.`) + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + if err := inputDef.Validate(); err != nil { + return err + } + + // Encode InputDefinition to its internal representation. + def := inputDef.Encode() + def.Name = inputDefName + + // Create InputDefinition. + if _, err := index.CreateInputDefinition(def); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.CreateInputDefinitionMessage{ + Index: indexName, + Definition: def, + }) + if err != nil { + api.Logger.Printf("problem sending CreateInputDefinition message: %s", err) + } + return nil +} + +// InputDefinition is deprecated and will be removed. +func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + if err := api.validate(apiInputDefinition); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + inputDef, err := index.InputDefinition(inputDefName) + if err != nil { + return nil, err + } + return inputDef, nil +} + +// DeleteInputDefinition is deprecated and will be removed. +func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + if err := api.validate(apiDeleteInputDefinition); err != nil { + return errors.Wrap(err, "validate api method") + } + + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete input definition from the index. + if err := index.DeleteInputDefinition(inputDefName); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.DeleteInputDefinitionMessage{ + Index: indexName, + Name: inputDefName, + }) + if err != nil { + api.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) + } + return nil +} + +// WriteInput is deprecated and will be removed. +func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + if err := api.validate(apiWriteInput); err != nil { + return errors.Wrap(err, "validate api method") + } + + api.Logger.Printf("WriteInput is deprecated and will be removed.") + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + for _, req := range reqs { + bits, err := api.inputJSONDataParser(req.(map[string]interface{}), index, inputDefName) + if err != nil { + return err + } + for fr, bs := range bits { + if err := index.InputBits(fr, bs); err != nil { + return err + } + } + } + + return nil +} + +// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. +func (api *API) RecalculateCaches(ctx context.Context) error { + if err := api.validate(apiRecalculateCaches); err != nil { + return errors.Wrap(err, "validate api method") + } + + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) + if err != nil { + return errors.Wrap(err, "broacasting message") + } + api.Holder.RecalculateCaches() + return nil +} + +// PostClusterMessage is for internal use. It decodes a protobuf message out of +// the body and forwards it to the BroadcastHandler. +func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { + if err := api.validate(apiClusterMessage); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Read entire body. + body, err := ioutil.ReadAll(reqBody) + if err != nil { + return errors.Wrap(err, "reading body") + } + + // Marshal into request object. + pb, err := UnmarshalMessage(body) + if err != nil { + return errors.Wrap(err, "unmarshaling message") + } + + // Forward the error message. + if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { + return errors.Wrap(err, "receiving message") + } + return nil +} + +// LocalID returns the current node's ID. +func (api *API) LocalID() string { + return api.Cluster.Node.ID +} + +// Schema returns information about each index in Pilosa including which frames +// and views they contain. +func (api *API) Schema(ctx context.Context) []*IndexInfo { + return api.Holder.Schema() +} + +// CreateField creates a new BSI field in the given index and frame. +func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { + if err := api.validate(apiCreateField); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Create new field. + if err := f.CreateField(field); err != nil { + return err + } + + // Send the create field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.CreateFieldMessage{ + Index: indexName, + Frame: frameName, + Field: encodeField(field), + }) + if err != nil { + api.Logger.Printf("problem sending CreateField message: %s", err) + } + return err +} + +// DeleteField deletes the given field. +func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete field. + if err := f.DeleteField(fieldName); err != nil { + return err + } + + // Send the delete field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFieldMessage{ + Index: indexName, + Frame: frameName, + Field: fieldName, + }) + if err != nil { + api.Logger.Printf("problem sending DeleteField message: %s", err) + } + return err +} + +// Fields returns the fields in the given frame. +func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { + if err := api.validate(apiFields); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + index := api.Holder.index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + frame := index.frame(frameName) + if frame == nil { + return nil, ErrFrameNotFound + } + + return frame.GetFields() +} + +// Views returns the views in the given frame. +func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { + if err := api.validate(apiViews); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Retrieve views. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Fetch views. + views := f.Views() + return views, nil +} + +// DeleteView removes the given view. +func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + if err := api.validate(apiDeleteView); err != nil { + return errors.Wrap(err, "validate api method") + } + + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete the view. + if err := f.DeleteView(viewName); err != nil { + // Ignore this error because views do not exist on all nodes due to slice distribution. + if err != ErrInvalidView { + return err + } + } + + // Send the delete view message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteViewMessage{ + Index: indexName, + Frame: frameName, + View: viewName, + }) + if err != nil { + api.Logger.Printf("problem sending DeleteView message: %s", err) + } + + return err +} + +// IndexAttrDiff +func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiIndexAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Retrieve index from holder. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Retrieve local blocks. + localBlocks, err := index.ColumnAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := index.ColumnAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFrameAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + // Retrieve index from holder. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Retrieve local blocks. + localBlocks, err := f.RowAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := f.RowAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +// Import bulk imports data into a particular index,frame,slice. +func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + if err := api.validate(apiImport); err != nil { + return errors.Wrap(err, "validate api method") + } + + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Convert timestamps to time.Time. + timestamps := make([]*time.Time, len(req.Timestamps)) + for i, ts := range req.Timestamps { + if ts == 0 { + continue + } + t := time.Unix(0, ts) + timestamps[i] = &t + } + + // Import into fragment. + err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + if err != nil { + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + } + return err +} + +// ImportValue bulk imports values into a particular field. +func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + if err := api.validate(apiImportValue); err != nil { + return errors.Wrap(err, "validate api method") + } + + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Import into fragment. + err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + if err != nil { + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + } + return err +} + +// MaxSlices returns the maximum slice number for each index in a map. +func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { + return api.Holder.MaxSlices() +} + +// MaxInverseSlices returns the maximum inverse slice number for each index in a +// map. +func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { + return api.Holder.MaxInverseSlices() +} + +// StatsWithTags returns an instance of whatever implementation of StatsClient +// pilosa is using with the given tags. +func (api *API) StatsWithTags(tags []string) StatsClient { + if api.Holder == nil || api.Cluster == nil { + return nil + } + return api.Holder.Stats.WithTags(tags...) +} + +// LongQueryTime returns the configured threshold for logging/statting +// long running queries. +func (api *API) LongQueryTime() time.Duration { + if api.Cluster == nil { + return 0 + } + return api.Cluster.LongQueryTime +} + +func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { + // Validate that this handler owns the slice. + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) + return nil, nil, ErrClusterDoesNotOwnSlice + } + + // Find the Index. + api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) + index := api.Holder.Index(indexName) + if index == nil { + api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + return nil, nil, ErrIndexNotFound + } + + // Retrieve frame. + frame := index.Frame(frameName) + if frame == nil { + api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) + return nil, nil, ErrFrameNotFound + } + return index, frame, nil +} + +// inputJSONDataParser validates input json file and executes SetBit. Deprecated - remove with input definition stuff. +func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { + inputDef, err := index.InputDefinition(name) + if err != nil { + return nil, err + } + // If field in input data is not in defined definition, return error. + var colValue uint64 + validFields := make(map[string]bool) + timestampFrame := make(map[string]int64) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + if field.PrimaryKey { + value, ok := req[field.Name] + if !ok { + return nil, fmt.Errorf("primary key does not exist") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) + } + // Find frame that need to add timestamp. + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } + } + } + } + + for key := range req { + _, ok := validFields[key] + if !ok { + return nil, fmt.Errorf("field not found: %s", key) + } + } + + setBits := make(map[string][]*Bit) + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + + // Looking into timestampFrame map and set timestamp to the whole frame + for _, action := range field.Actions { + frame := action.Frame + timestamp := timestampFrame[action.Frame] + // Skip input data field values that are set to null + if req[field.Name] == nil { + continue + } + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) + if err != nil { + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) + } + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } + } + } + return setBits, nil +} + +// SetCoordinator makes a new Node the cluster coordinator. +func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + if err := api.validate(apiSetCoordinator); err != nil { + return nil, nil, errors.Wrap(err, "validate api method") + } + + oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.NodeByID(id) + if newNode == nil { + return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") + } + + // If the new coordinator is this node, do the SetCoordinator directly. + if newNode.ID == api.LocalID() { + return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + } + + // Send the set-coordinator message to new node. + err = api.Broadcaster.SendTo( + newNode, + &internal.SetCoordinatorMessage{ + New: EncodeNode(newNode), + }) + if err != nil { + return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) + } + return oldNode, newNode, nil +} + +// RemoveNode puts the cluster into the "RESIZING" state and begins the job of +// removing the given node. +func (api *API) RemoveNode(id string) (*Node, error) { + if err := api.validate(apiRemoveNode); err != nil { + return nil, errors.Wrap(err, "validate api method") + } + + removeNode := api.Cluster.nodeByID(id) + if removeNode == nil { + return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") + } + + // Start the resize process (similar to NodeJoin) + err := api.Cluster.NodeLeave(removeNode) + if err != nil { + return removeNode, errors.Wrap(err, "calling node leave") + } + return removeNode, nil +} + +// ResizeAbort stops the current resize job. +func (api *API) ResizeAbort() error { + if err := api.validate(apiResizeAbort); err != nil { + return errors.Wrap(err, "validate api method") + } + + err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + return errors.Wrap(err, "complete current job") +} + +// State returns the cluster state which is usually "NORMAL", but could be +// "STARTING", "RESIZING", or potentially others. See cluster.go for more +// details. +func (api *API) State() string { + return api.Cluster.State() +} + +// Version returns the Pilosa version. +func (api *API) Version() string { + return strings.TrimPrefix(Version, "v") +} + +// Info returns information about this server instance +func (api *API) Info() ServerInfo { + return ServerInfo{ + SliceWidth: SliceWidth, + } +} + +type ServerInfo struct { + SliceWidth uint64 `json:"sliceWidth"` +} + +type apiMethod int + +// API validation constants. +const ( + apiClusterMessage apiMethod = iota + apiCreateField + apiCreateFrame + apiCreateIndex + apiCreateInputDefinition + apiDeleteField + apiDeleteFrame + apiDeleteIndex + apiDeleteInputDefinition + apiDeleteView + apiExportCSV + apiFields + apiFragmentBlockData + apiFragmentBlocks + apiFrameAttrDiff + //apiHosts // not implemented + apiImport + apiImportValue + apiIndex + apiIndexAttrDiff + apiInputDefinition + //apiLocalID // not implemented + //apiLongQueryTime // not implemented + apiMarshalFragment + //apiMaxInverseSlices // not implemented + //apiMaxSlices // not implemented + apiQuery + apiRecalculateCaches + apiRemoveNode + apiResizeAbort + apiRestoreFrame + //apiSchema // not implemented + apiSetCoordinator + apiSliceNodes + //apiState // not implemented + //apiStatsWithTags // not implemented + apiUnmarshalFragment + //apiVersion // not implemented + apiViews + apiWriteInput +) + +var methodsCommon = map[apiMethod]struct{}{ + apiClusterMessage: struct{}{}, + apiMarshalFragment: struct{}{}, + apiSetCoordinator: struct{}{}, +} + +var methodsResizing = map[apiMethod]struct{}{ + apiResizeAbort: struct{}{}, +} + +var methodsNormal = map[apiMethod]struct{}{ + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, +} diff --git a/apimethod_string.go b/apimethod_string.go new file mode 100644 index 000000000..8ee574f8d --- /dev/null +++ b/apimethod_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiMethod"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460} + +func (i apiMethod) String() string { + if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { + return fmt.Sprintf("apiMethod(%d)", i) + } + return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] +} diff --git a/attr.go b/attr.go index ab29fc9c0..03ea4f43f 100644 --- a/attr.go +++ b/attr.go @@ -16,21 +16,12 @@ package pilosa import ( "bytes" - "crypto/sha1" - "encoding/binary" - "fmt" "sort" - "sync" - "time" - "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) -// AttrBlockSize is the size of attribute blocks for anti-entropy. -const AttrBlockSize = 100 - // Attribute data type enum. const ( AttrTypeString = 1 @@ -39,313 +30,111 @@ const ( AttrTypeFloat = 4 ) -// AttrCache represents a cache for attributes. -type AttrCache struct { - mu sync.RWMutex - attrs map[uint64]map[string]interface{} +// AttrStore represents an interface for handling row/column attributes. +type AttrStore interface { + Path() string + Open() error + Close() error + Attrs(id uint64) (m map[string]interface{}, err error) + SetAttrs(id uint64, m map[string]interface{}) error + SetBulkAttrs(m map[uint64]map[string]interface{}) error + Blocks() ([]AttrBlock, error) + BlockData(i uint64) (map[uint64]map[string]interface{}, error) } -// Get returns the cached attributes for a given id. -func (c *AttrCache) Get(id uint64) map[string]interface{} { - c.mu.RLock() - defer c.mu.RUnlock() - attrs := c.attrs[id] - if attrs == nil { - return nil - } - - // Make a copy for safety - ret := make(map[string]interface{}) - for k, v := range attrs { - ret[k] = v - } - return ret +func init() { + NopAttrStore = &nopAttrStore{} } -// Set updates the cached attributes for a given id. -func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { - c.mu.Lock() - defer c.mu.Unlock() - c.attrs[id] = attrs +// NopAttrStore represents an AttrStore that doesn't do anything. +var NopAttrStore AttrStore + +func NewNopAttrStore(string) AttrStore { + return &nopAttrStore{} } -// AttrStore represents a storage layer for attributes. -type AttrStore struct { - mu sync.RWMutex - path string - db *bolt.DB - attrCache *AttrCache -} +// nopAttrStore represents a no-op implementation of the AttrStore interface. +type nopAttrStore struct{} -// NewAttrCache returns a new instance of AttrCache. -func NewAttrCache() *AttrCache { - return &AttrCache{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(path string) *AttrStore { - return &AttrStore{ - path: path, - attrCache: NewAttrCache(), - } -} - -// Path returns path to the store's data file. -func (s *AttrStore) Path() string { return s.path } - -// Open opens and initializes the store. -func (s *AttrStore) Open() error { - // Open storage. - db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return err - } - s.db = db - - // Initialize database. - if err := s.db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil { - return err - } - return nil - }); err != nil { - return err - } +// Path is a no-op implementation of AttrStore Path method. +func (s *nopAttrStore) Path() string { return "" } +// Open is a no-op implementation of AttrStore Open method. +func (s *nopAttrStore) Open() error { return nil } -// Close closes the store. -func (s *AttrStore) Close() error { - if s.db != nil { - s.db.Close() - } +// Close is a no-op implementation of AttrStore Close method. +func (s *nopAttrStore) Close() error { return nil } -// Attrs returns a set of attributes by ID. -func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { - s.mu.RLock() - defer s.mu.RUnlock() - - // Check cache for map. - if m = s.attrCache.Get(id); m != nil { - return m, nil - } - - // Find attributes from storage. - if err = s.db.View(func(tx *bolt.Tx) error { - m, err = txAttrs(tx, id) - if err != nil { - return err - } - return nil - }); err != nil { - return nil, err - } - - // Add to cache. - s.attrCache.Set(id, m) - - return +// Attrs is a no-op implementation of AttrStore Attrs method. +func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return nil, nil } -// SetAttrs sets attribute values for a given ID. -func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { - // Ignore empty maps. - if len(m) == 0 { - return nil - } - - // Check if the attributes already exist under a read-only lock. - if attr, err := s.Attrs(id); err != nil { - return err - } else if attr != nil && mapContains(attr, m) { - return nil - } - - // Obtain write lock. - s.mu.Lock() - defer s.mu.Unlock() - - var attr map[string]interface{} - if err := s.db.Update(func(tx *bolt.Tx) error { - tmp, err := txUpdateAttrs(tx, id, m) - if err != nil { - return err - } - attr = tmp - - return nil - }); err != nil { - return err - } - - // Swap attributes map in cache. - s.attrCache.Set(id, attr) - +// SetAttrs is a no-op implementation of AttrStore SetAttrs method. +func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil } -// SetBulkAttrs sets attribute values for a set of ids. -func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { - s.mu.Lock() - defer s.mu.Unlock() +// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method. +func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + return nil +} - attrs := make(map[uint64]map[string]interface{}) - if err := s.db.Update(func(tx *bolt.Tx) error { - // Collect and sort keys. - ids := make([]uint64, 0, len(m)) - for id := range m { - ids = append(ids, id) +// Blocks is a no-op implementation of AttrStore Blocks method. +func (s *nopAttrStore) Blocks() ([]AttrBlock, error) { + return nil, nil +} + +// BlockData is a no-op implementation of AttrStore BlockData method. +func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} + +// AttrBlock represents a checksummed block of the attribute store. +type AttrBlock struct { + ID uint64 `json:"id"` + Checksum []byte `json:"checksum"` +} + +// AttrBlocks represents a list of blocks. +type AttrBlocks []AttrBlock + +// Diff returns a list of block ids that are different or are new in other. +// Block lists must be in sorted order. +func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { + var ids []uint64 + for { + // Read next block from each list. + var blk0, blk1 *AttrBlock + if len(a) > 0 { + blk0 = &a[0] + } + if len(other) > 0 { + blk1 = &other[0] } - sort.Sort(uint64Slice(ids)) - // Update attributes for each id. - for _, id := range ids { - attr, err := txUpdateAttrs(tx, id, m[id]) - if err != nil { - return err + // Exit if "a" contains no more blocks. + if blk0 == nil { + return ids + } + + // Add block ID if it's different or if it's only in "a". + if blk1 == nil || blk0.ID < blk1.ID { + ids = append(ids, blk0.ID) + a = a[1:] + } else if blk1.ID < blk0.ID { + other = other[1:] + } else { + if !bytes.Equal(blk0.Checksum, blk1.Checksum) { + ids = append(ids, blk0.ID) } - attrs[id] = attr - } - - return nil - }); err != nil { - return err - } - - // Swap attributes map in cache. - for id, attr := range attrs { - s.attrCache.Set(id, attr) - } - - return nil -} - -// Blocks returns a list of all blocks in the store. -func (s *AttrStore) Blocks() ([]AttrBlock, error) { - tx, err := s.db.Begin(false) - if err != nil { - return nil, err - } - defer tx.Rollback() - - // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) - - // Iterate over each block. - var blocks []AttrBlock - for cur.nextBlock() { - block := AttrBlock{ID: cur.blockID()} - - // Compute checksum of every key/value in block. - h := sha1.New() - for k, v := cur.next(); k != nil; k, v = cur.next() { - h.Write(k) - h.Write(v) - } - block.Checksum = h.Sum(nil) - - // Append block. - blocks = append(blocks, block) - } - - return blocks, nil -} - -// BlockData returns all data for a single block. -func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { - m := make(map[uint64]map[string]interface{}) - - // Start read-only transaction. - tx, err := s.db.Begin(false) - if err != nil { - return nil, err - } - defer tx.Rollback() - - // Move to the start of the block. - min := u64tob(uint64(i) * AttrBlockSize) - max := u64tob(uint64(i+1) * AttrBlockSize) - cur := tx.Bucket([]byte("attrs")).Cursor() - for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { - // Exit if we're past the end of the block. - if bytes.Compare(k, max) != -1 { - break - } - - // Decode attribute map and associate with id. - var pb internal.AttrMap - if err := proto.Unmarshal(v, &pb); err != nil { - return nil, err - } - m[btou64(k)] = decodeAttrs(pb.GetAttrs()) - } - - return m, nil -} - -// txAttrs returns a map of attributes for an id. -func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { - v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) - if v == nil { - return emptyMap, nil - } - - var pb internal.AttrMap - if err := proto.Unmarshal(v, &pb); err != nil { - return nil, err - } - return decodeAttrs(pb.GetAttrs()), nil -} - -// txUpdateAttrs updates the attributes for an id. -// Returns the new combined set of attributes for the id. -func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) { - attr, err := txAttrs(tx, id) - if err != nil { - return nil, err - } - - // Create a new map if it is empty so we don't update emptyMap. - if len(attr) == 0 { - attr = make(map[string]interface{}, len(m)) - } - - // Merge attributes with original values. - // Nil values should delete keys. - for k, v := range m { - if v == nil { - delete(attr, k) - continue - } - - switch v := v.(type) { - case int: - attr[k] = int64(v) - case uint: - attr[k] = int64(v) - case uint64: - attr[k] = int64(v) - case string, int64, bool, float64: - attr[k] = v - default: - return nil, fmt.Errorf("invalid attr type: %T", v) + a, other = a[1:], other[1:] } } - - // Marshal and save new values. - buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) - if err != nil { - return nil, err - } - if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil { - return nil, err - } - return attr, nil } func encodeAttrs(m map[string]interface{}) []*internal.Attr { @@ -419,136 +208,16 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} { return other } -// u64tob encodes v to big endian encoding. -func u64tob(v uint64) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, v) - return b +// EncodeAttrs encodes an attribute map into a byte slice. +func EncodeAttrs(attr map[string]interface{}) ([]byte, error) { + return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) } -// btou64 decodes b from big endian encoding. -func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } - -// emptyMap is a reusable map that contains no keys. -var emptyMap = make(map[string]interface{}) - -// AttrBlock represents a checksummed block of the attribute store. -type AttrBlock struct { - ID uint64 `json:"id"` - Checksum []byte `json:"checksum"` -} - -// AttrBlocks represents a list of blocks. -type AttrBlocks []AttrBlock - -// Diff returns a list of block ids that are different or are new in other. -// Block lists must be in sorted order. -func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { - var ids []uint64 - for { - // Read next block from each list. - var blk0, blk1 *AttrBlock - if len(a) > 0 { - blk0 = &a[0] - } - if len(other) > 0 { - blk1 = &other[0] - } - - // Exit if "a" contains no more blocks. - if blk0 == nil { - return ids - } - - // Add block ID if it's different or if it's only in "a". - if blk1 == nil || blk0.ID < blk1.ID { - ids = append(ids, blk0.ID) - a = a[1:] - } else if blk1.ID < blk0.ID { - other = other[1:] - } else { - if !bytes.Equal(blk0.Checksum, blk1.Checksum) { - ids = append(ids, blk0.ID) - } - a, other = a[1:], other[1:] - } - } -} - -// blockCursor represents a cursor for iterating over blocks of a bolt bucket. -type blockCursor struct { - cur *bolt.Cursor - base uint64 - n uint64 - - buf struct { - key []byte - value []byte - filled bool - } -} - -// newBlockCursor returns a new block cursor that wraps cur using n sized blocks. -func newBlockCursor(c *bolt.Cursor, n int) blockCursor { - cur := blockCursor{ - cur: c, - n: uint64(n), - } - cur.buf.key, cur.buf.value = c.First() - cur.buf.filled = true - return cur -} - -// blockID returns the current block ID. Only valid after call to nextBlock(). -func (cur *blockCursor) blockID() uint64 { return cur.base } - -// nextBlock moves the cursor to the next block. -// Returns true if another block exists, otherwise returns false. -func (cur *blockCursor) nextBlock() bool { - if cur.buf.key == nil { - return false - } - - cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n - return true -} - -// next returns the next key/value within the block. -// Returns nils at the end of the block. -func (cur *blockCursor) next() (key, value []byte) { - // Use buffered value, if set. - if cur.buf.filled { - key, value = cur.buf.key, cur.buf.value - cur.buf.filled = false - return key, value - } - - // Read next key. - key, value = cur.cur.Next() - - // Fill buffer for EOF. - if key == nil { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false - return nil, nil - } - - // Parse key and buffer if outside of block. - id := binary.BigEndian.Uint64(key) - if id/cur.n > cur.base { - cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true - return nil, nil - } - - return key, value -} - -// mapContains returns true if all keys & values of subset are in m. -func mapContains(m, subset map[string]interface{}) bool { - for k, v := range subset { - value, ok := m[k] - if !ok || value != v { - return false - } - } - return true +// DecodeAttrs decodes a byte slice into an attribute map. +func DecodeAttrs(v []byte) (map[string]interface{}, error) { + var pb internal.AttrMap + if err := proto.Unmarshal(v, &pb); err != nil { + return nil, err + } + return decodeAttrs(pb.GetAttrs()), nil } diff --git a/bitmap.go b/bitmap.go index 0728ec218..975b4fb03 100644 --- a/bitmap.go +++ b/bitmap.go @@ -14,8 +14,6 @@ package pilosa -// #cgo CFLAGS:-mpopcnt - import ( "encoding/json" "sort" diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go new file mode 100644 index 000000000..ebb539903 --- /dev/null +++ b/boltdb/attrstore.go @@ -0,0 +1,465 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "bytes" + + "encoding/binary" + "fmt" + "sort" + "sync" + "time" + + "github.com/cespare/xxhash" + + "github.com/boltdb/bolt" + "github.com/pilosa/pilosa" +) + +// AttrBlockSize is the size of attribute blocks for anti-entropy. +const AttrBlockSize = 100 + +// AttrCache represents a cache for attributes. +type AttrCache struct { + mu sync.RWMutex + attrs map[uint64]map[string]interface{} +} + +// Get returns the cached attributes for a given id. +func (c *AttrCache) Get(id uint64) map[string]interface{} { + c.mu.RLock() + defer c.mu.RUnlock() + attrs := c.attrs[id] + if attrs == nil { + return nil + } + + // Make a copy for safety + ret := make(map[string]interface{}) + for k, v := range attrs { + ret[k] = v + } + return ret +} + +// Set updates the cached attributes for a given id. +func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { + c.mu.Lock() + defer c.mu.Unlock() + c.attrs[id] = attrs +} + +// AttrStore represents a storage layer for attributes. +type AttrStore struct { + mu sync.RWMutex + path string + db *bolt.DB + attrCache *AttrCache +} + +// NewAttrCache returns a new instance of AttrCache. +func NewAttrCache() *AttrCache { + return &AttrCache{ + attrs: make(map[uint64]map[string]interface{}), + } +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(path string) pilosa.AttrStore { + return &AttrStore{ + path: path, + attrCache: NewAttrCache(), + } +} + +// Path returns path to the store's data file. +func (s *AttrStore) Path() string { return s.path } + +// Open opens and initializes the store. +func (s *AttrStore) Open() error { + // Open storage. + db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return err + } + s.db = db + + // Initialize database. + if err := s.db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + return nil +} + +// Close closes the store. +func (s *AttrStore) Close() error { + if s.db != nil { + s.db.Close() + } + return nil +} + +// Attrs returns a set of attributes by ID. +func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + // Check cache for map. + if m = s.attrCache.Get(id); m != nil { + return m, nil + } + + // Find attributes from storage. + if err = s.db.View(func(tx *bolt.Tx) error { + m, err = txAttrs(tx, id) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + + // Add to cache. + s.attrCache.Set(id, m) + + return +} + +// SetAttrs sets attribute values for a given ID. +func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + // Ignore empty maps. + if len(m) == 0 { + return nil + } + + // Check if the attributes already exist under a read-only lock. + if attr, err := s.Attrs(id); err != nil { + return err + } else if attr != nil && mapContains(attr, m) { + return nil + } + + // Obtain write lock. + s.mu.Lock() + defer s.mu.Unlock() + + var attr map[string]interface{} + if err := s.db.Update(func(tx *bolt.Tx) error { + tmp, err := txUpdateAttrs(tx, id, m) + if err != nil { + return err + } + attr = tmp + + return nil + }); err != nil { + return err + } + + // Swap attributes map in cache. + s.attrCache.Set(id, attr) + + return nil +} + +// SetBulkAttrs sets attribute values for a set of ids. +func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + s.mu.Lock() + defer s.mu.Unlock() + + attrs := make(map[uint64]map[string]interface{}) + if err := s.db.Update(func(tx *bolt.Tx) error { + // Collect and sort keys. + ids := make([]uint64, 0, len(m)) + for id := range m { + ids = append(ids, id) + } + sort.Sort(uint64Slice(ids)) + + // Update attributes for each id. + for _, id := range ids { + attr, err := txUpdateAttrs(tx, id, m[id]) + if err != nil { + return err + } + attrs[id] = attr + } + + return nil + }); err != nil { + return err + } + + // Swap attributes map in cache. + for id, attr := range attrs { + s.attrCache.Set(id, attr) + } + + return nil +} + +// Blocks returns a list of all blocks in the store. +func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { + tx, err := s.db.Begin(false) + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Wrap cursor to segment by block. + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) + + // Iterate over each block. + var blocks []pilosa.AttrBlock + for cur.nextBlock() { + block := pilosa.AttrBlock{ID: cur.blockID()} + + // Compute checksum of every key/value in block. + h := xxhash.New() + for k, v := cur.next(); k != nil; k, v = cur.next() { + h.Write(k) + h.Write(v) + } + block.Checksum = h.Sum(nil) + + // Append block. + blocks = append(blocks, block) + } + + return blocks, nil +} + +// BlockData returns all data for a single block. +func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + m := make(map[uint64]map[string]interface{}) + + // Start read-only transaction. + tx, err := s.db.Begin(false) + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Move to the start of the block. + min := u64tob(uint64(i) * AttrBlockSize) + max := u64tob(uint64(i+1) * AttrBlockSize) + cur := tx.Bucket([]byte("attrs")).Cursor() + for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { + // Exit if we're past the end of the block. + if bytes.Compare(k, max) != -1 { + break + } + + // Decode attribute map and associate with id. + attrs, err := pilosa.DecodeAttrs(v) + if err != nil { + return nil, err + } + m[btou64(k)] = attrs + + } + + return m, nil +} + +// txAttrs returns a map of attributes for an id. +func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { + v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) + if v == nil { + return emptyMap, nil + } + return pilosa.DecodeAttrs(v) +} + +// txUpdateAttrs updates the attributes for an id. +// Returns the new combined set of attributes for the id. +func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) { + attr, err := txAttrs(tx, id) + if err != nil { + return nil, err + } + + // Create a new map if it is empty so we don't update emptyMap. + if len(attr) == 0 { + attr = make(map[string]interface{}, len(m)) + } + + // Merge attributes with original values. + // Nil values should delete keys. + for k, v := range m { + if v == nil { + delete(attr, k) + continue + } + + switch v := v.(type) { + case int: + attr[k] = int64(v) + case uint: + attr[k] = int64(v) + case uint64: + attr[k] = int64(v) + case string, int64, bool, float64: + attr[k] = v + default: + return nil, fmt.Errorf("invalid attr type: %T", v) + } + } + + // Marshal and save new values. + buf, err := pilosa.EncodeAttrs(attr) + if err != nil { + return nil, err + } + if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil { + return nil, err + } + return attr, nil +} + +// u64tob encodes v to big endian encoding. +func u64tob(v uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, v) + return b +} + +// btou64 decodes b from big endian encoding. +func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } + +// emptyMap is a reusable map that contains no keys. +var emptyMap = make(map[string]interface{}) + +// mapContains returns true if all keys & values of subset are in m. +func mapContains(m, subset map[string]interface{}) bool { + for k, v := range subset { + value, ok := m[k] + if !ok || value != v { + return false + } + } + return true +} + +// uint64Slice represents a sortable slice of uint64 numbers. +type uint64Slice []uint64 + +func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p uint64Slice) Len() int { return len(p) } +func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } + +// merge combines p and other to a unique sorted set of values. +// p and other must both have unique sets and be sorted. +func (p uint64Slice) merge(other []uint64) []uint64 { + ret := make([]uint64, 0, len(p)) + + i, j := 0, 0 + for i < len(p) && j < len(other) { + a, b := p[i], other[j] + if a == b { + ret = append(ret, a) + i, j = i+1, j+1 + } else if a < b { + ret = append(ret, a) + i++ + } else { + ret = append(ret, b) + j++ + } + } + + if i < len(p) { + ret = append(ret, p[i:]...) + } else if j < len(other) { + ret = append(ret, other[j:]...) + } + + return ret +} + +// blockCursor represents a cursor for iterating over blocks of a bolt bucket. +type blockCursor struct { + cur *bolt.Cursor + base uint64 + n uint64 + + buf struct { + key []byte + value []byte + filled bool + } +} + +// newBlockCursor returns a new block cursor that wraps cur using n sized blocks. +func newBlockCursor(c *bolt.Cursor, n int) blockCursor { + cur := blockCursor{ + cur: c, + n: uint64(n), + } + cur.buf.key, cur.buf.value = c.First() + cur.buf.filled = true + return cur +} + +// blockID returns the current block ID. Only valid after call to nextBlock(). +func (cur *blockCursor) blockID() uint64 { return cur.base } + +// nextBlock moves the cursor to the next block. +// Returns true if another block exists, otherwise returns false. +func (cur *blockCursor) nextBlock() bool { + if cur.buf.key == nil { + return false + } + + cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n + return true +} + +// next returns the next key/value within the block. +// Returns nils at the end of the block. +func (cur *blockCursor) next() (key, value []byte) { + // Use buffered value, if set. + if cur.buf.filled { + key, value = cur.buf.key, cur.buf.value + cur.buf.filled = false + return key, value + } + + // Read next key. + key, value = cur.cur.Next() + + // Fill buffer for EOF. + if key == nil { + cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false + return nil, nil + } + + // Parse key and buffer if outside of block. + id := binary.BigEndian.Uint64(key) + if id/cur.n > cur.base { + cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true + return nil, nil + } + + return key, value +} diff --git a/broadcast.go b/broadcast.go index 38eb53b96..de43f3b85 100644 --- a/broadcast.go +++ b/broadcast.go @@ -22,38 +22,27 @@ import ( "github.com/pilosa/pilosa/internal" ) -// NodeSet represents an interface for Node membership and inter-node communication. -type NodeSet interface { - // Returns a list of all Nodes in the cluster - Nodes() []*Node - - // Open starts any network activity implemented by the NodeSet - Open() error +// MemberSet represents an interface for Node membership and inter-node communication. +type MemberSet interface { + // Open starts any network activity implemented by the MemberSet + // Node is the local node, used for membership broadcasts. + Open(n *Node) error } -// StaticNodeSet represents a basic NodeSet for testing. -type StaticNodeSet struct { +// StaticMemberSet represents a basic MemberSet for testing. +type StaticMemberSet struct { nodes []*Node } -// NewStaticNodeSet creates a statically defined NodeSet. -func NewStaticNodeSet() *StaticNodeSet { - return &StaticNodeSet{} +// NewStaticMemberSet creates a statically defined MemberSet. +func NewStaticMemberSet(nodes []*Node) *StaticMemberSet { + return &StaticMemberSet{ + nodes: nodes, + } } -// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. -func (s *StaticNodeSet) Nodes() []*Node { - return s.nodes -} - -// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing. -func (s *StaticNodeSet) Open() error { - return nil -} - -// Join sets the NodeSet nodes to the slice of Nodes passed in. -func (s *StaticNodeSet) Join(nodes []*Node) error { - s.nodes = nodes +// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. +func (s *StaticMemberSet) Open(n *Node) error { return nil } @@ -61,6 +50,7 @@ func (s *StaticNodeSet) Join(nodes []*Node) error { type Broadcaster interface { SendSync(pb proto.Message) error SendAsync(pb proto.Message) error + SendTo(to *Node, pb proto.Message) error } func init() { @@ -73,16 +63,21 @@ var NopBroadcaster Broadcaster type nopBroadcaster struct{} -// SendSync A no-op implemenetation of Broadcaster SendSync method. +// SendSync A no-op implementation of Broadcaster SendSync method. func (n *nopBroadcaster) SendSync(pb proto.Message) error { return nil } -// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +// SendAsync A no-op implementation of Broadcaster SendAsync method. func (n *nopBroadcaster) SendAsync(pb proto.Message) error { return nil } +// SendTo is a no-op implementation of Broadcaster SendTo method. +func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error { + return nil +} + // BroadcastHandler is the interface for the pilosa object which knows how to // handle broadcast messages. (Hint: this is implemented by pilosa.Server) type BroadcastHandler interface { @@ -117,21 +112,32 @@ var NopGossiper Gossiper type nopGossiper struct{} -// SendAsync A no-op implemenetation of Gossiper SendAsync method. +// SendAsync A no-op implementation of Gossiper SendAsync method. func (n *nopGossiper) SendAsync(pb proto.Message) error { return nil } // Broadcast message types. const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateIndex = 2 - MessageTypeDeleteIndex = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 - MessageTypeCreateInputDefinition = 6 - MessageTypeDeleteInputDefinition = 7 - MessageTypeDeleteView = 8 + MessageTypeCreateSlice = iota + MessageTypeCreateIndex + MessageTypeDeleteIndex + MessageTypeCreateFrame + MessageTypeDeleteFrame + MessageTypeCreateView + MessageTypeDeleteView + MessageTypeCreateField + MessageTypeDeleteField + MessageTypeCreateInputDefinition + MessageTypeDeleteInputDefinition + MessageTypeClusterStatus + MessageTypeResizeInstruction + MessageTypeResizeInstructionComplete + MessageTypeSetCoordinator + MessageTypeUpdateCoordinator + MessageTypeNodeState + MessageTypeRecalculateCaches + MessageTypeNodeEvent ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -148,12 +154,34 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateFrame case *internal.DeleteFrameMessage: typ = MessageTypeDeleteFrame + case *internal.CreateViewMessage: + typ = MessageTypeCreateView + case *internal.DeleteViewMessage: + typ = MessageTypeDeleteView + case *internal.CreateFieldMessage: + typ = MessageTypeCreateField + case *internal.DeleteFieldMessage: + typ = MessageTypeDeleteField case *internal.CreateInputDefinitionMessage: typ = MessageTypeCreateInputDefinition case *internal.DeleteInputDefinitionMessage: typ = MessageTypeDeleteInputDefinition - case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView + case *internal.ClusterStatus: + typ = MessageTypeClusterStatus + case *internal.ResizeInstruction: + typ = MessageTypeResizeInstruction + case *internal.ResizeInstructionComplete: + typ = MessageTypeResizeInstructionComplete + case *internal.SetCoordinatorMessage: + typ = MessageTypeSetCoordinator + case *internal.UpdateCoordinatorMessage: + typ = MessageTypeUpdateCoordinator + case *internal.NodeStateMessage: + typ = MessageTypeNodeState + case *internal.RecalculateCaches: + typ = MessageTypeRecalculateCaches + case *internal.NodeEventMessage: + typ = MessageTypeNodeEvent default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -180,12 +208,34 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateFrameMessage{} case MessageTypeDeleteFrame: m = &internal.DeleteFrameMessage{} + case MessageTypeCreateView: + m = &internal.CreateViewMessage{} + case MessageTypeDeleteView: + m = &internal.DeleteViewMessage{} + case MessageTypeCreateField: + m = &internal.CreateFieldMessage{} + case MessageTypeDeleteField: + m = &internal.DeleteFieldMessage{} case MessageTypeCreateInputDefinition: m = &internal.CreateInputDefinitionMessage{} case MessageTypeDeleteInputDefinition: m = &internal.DeleteInputDefinitionMessage{} - case MessageTypeDeleteView: - m = &internal.DeleteViewMessage{} + case MessageTypeClusterStatus: + m = &internal.ClusterStatus{} + case MessageTypeResizeInstruction: + m = &internal.ResizeInstruction{} + case MessageTypeResizeInstructionComplete: + m = &internal.ResizeInstructionComplete{} + case MessageTypeSetCoordinator: + m = &internal.SetCoordinatorMessage{} + case MessageTypeUpdateCoordinator: + m = &internal.UpdateCoordinatorMessage{} + case MessageTypeNodeState: + m = &internal.NodeStateMessage{} + case MessageTypeRecalculateCaches: + m = &internal.RecalculateCaches{} + case MessageTypeNodeEvent: + m = &internal.NodeEventMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/broadcast_test.go b/broadcast_test.go index 8f0245c88..970a249cb 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -15,12 +15,16 @@ package pilosa_test import ( + "bytes" "reflect" "testing" + "io/ioutil" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" ) // Ensure a message can be marshaled and unmarshaled. @@ -52,8 +56,18 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { - - s := pilosa.NewServer() + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } + com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) + com.Config.Bind = "localhost:0" + com.Config.DataDir = path + err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO + if err != nil { + t.Fatalf("setting up server: %v", err) + } + s := com.Server sbr := NewSimpleBroadcastReceiver() sbh := NewSimpleBroadcastHandler() diff --git a/cache.go b/cache.go index 9c909ced4..e27d2f93c 100644 --- a/cache.go +++ b/cache.go @@ -22,8 +22,8 @@ import ( "sync" "time" - "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/lru" ) const ( diff --git a/client.go b/client.go index bbf118399..72225aef2 100644 --- a/client.go +++ b/client.go @@ -19,7 +19,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -35,6 +34,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" ) // ClientOptions represents the configuration for a InternalHTTPClient @@ -58,7 +58,7 @@ func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTT uri, err := NewURIFromAddress(host) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting URI") } client := NewInternalHTTPClientFromURI(uri, remoteClient) @@ -88,15 +88,12 @@ func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[st // maxSliceByIndex returns the number of slices on a server by index. func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.clientURI(ctx), "/slices/max") - u.RawQuery = (&url.Values{ - "inverse": {strconv.FormatBool(inverse)}, - }).Encode() + u := uriPathToURL(c.defaultURI, "/slices/max") // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -104,18 +101,21 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() - var rsp sliceMaxResponse + var rsp getSlicesMaxResponse if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("http: status=%d", resp.StatusCode) } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, fmt.Errorf("json decode: %s", err) } - return rsp.MaxSlices, nil + if inverse { + return rsp.Inverse, nil + } + return rsp.Standard, nil } // Schema returns all index and frame schema information. @@ -126,7 +126,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Build request. req, err := http.NewRequest("GET", u, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -134,7 +134,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -154,14 +154,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt Options: opt, }) if err != nil { - return err + return errors.Wrap(err, "encoding request") } // Create URL & HTTP request. u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") @@ -171,14 +171,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } // Handle response based on status code. @@ -201,7 +201,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -209,7 +209,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -223,8 +223,13 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl return a, nil } -// ExecuteQuery executes query against index on the server. -func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +// Query executes query against the index. +func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { if index == "" { return nil, ErrIndexRequired } else if queryRequest.Query == "" { @@ -234,14 +239,14 @@ func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, que // Encode request object. buf, err := proto.Marshal(queryRequest) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Create HTTP request. - u := c.clientURI(ctx).Path(fmt.Sprintf("/index/%s/query", index)) + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) @@ -252,14 +257,14 @@ func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, que // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return nil, errors.New(string(body)) } @@ -296,7 +301,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl // Import to each node. for _, node := range nodes { if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) } } @@ -317,13 +322,12 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, b } node := &Node{ - Scheme: c.defaultURI.Scheme(), - Host: c.defaultURI.HostPort(), + URI: *c.defaultURI, } // Import to node. if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) } return nil @@ -394,7 +398,7 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b u := nodePathToURL(node, "/import") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") @@ -404,14 +408,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return errors.New(string(body)) } @@ -448,7 +452,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel // Import to each node. for _, node := range nodes { if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.Host, err) + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) } } @@ -482,7 +486,7 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu u := nodePathToURL(node, "/import-value") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") @@ -492,14 +496,14 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body and unmarshal response. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } else if resp.StatusCode != http.StatusOK { return errors.New(string(body)) } @@ -536,7 +540,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s node := nodes[i] if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { return nil @@ -560,7 +564,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Generate HTTP request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Accept", "text/csv") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -568,7 +572,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -579,7 +583,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde // Copy body to writer. if _, err := io.Copy(w, resp.Body); err != nil { - return err + return errors.Wrap(err, "copying") } return nil @@ -614,13 +618,13 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f // Backup every slice to the tar file. for i := uint64(0); i <= maxSlices[index]; i++ { if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil { - return err + return errors.Wrap(err, "backing up slice") } } // Close tar file. if err := tw.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } return nil @@ -640,9 +644,9 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, // Read entire buffer to determine file size. data, err := ioutil.ReadAll(r) if err != nil { - return err + return errors.Wrap(err, "reading") } else if err := r.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } // Write slice file header. @@ -652,12 +656,12 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, Size: int64(len(data)), ModTime: time.Now(), }); err != nil { - return err + return errors.Wrap(err, "writing header") } // Write buffer to file. if _, err := tw.Write(data); err != nil { - return fmt.Errorf("write buffer: %s", err) + return errors.Wrap(err, "writing buffer") } return nil @@ -688,6 +692,13 @@ func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view return nil, fmt.Errorf("unable to connect to any owner") } +func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) { + node := &Node{ + URI: uri, + } + return c.backupSliceNode(ctx, index, frame, view, slice, node) +} + func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ @@ -700,7 +711,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -708,7 +719,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } // Return error if status is not OK. @@ -717,7 +728,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, return nil, ErrFragmentNotFound } else if resp.StatusCode != http.StatusOK { resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.Host, resp.StatusCode) + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) } return resp.Body, nil @@ -740,7 +751,7 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index if err == io.EOF { return nil } else if err != nil { - return err + return errors.Wrap(err, "opening") } // Parse slice from entry name. @@ -752,12 +763,12 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index // Read file into buffer. var buf bytes.Buffer if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil { - return err + return errors.Wrap(err, "copying") } // Restore file to all nodes that own it. if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil { - return err + return errors.Wrap(err, "restoring") } } } @@ -783,20 +794,20 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("User-Agent", "pilosa/"+Version) resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } resp.Body.Close() // Return error if response not OK. if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: host=%s, code=%d", node.Host, resp.StatusCode) + return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode) } } @@ -814,14 +825,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin Options: opt, }) if err != nil { - return err + return errors.Wrap(err, "marshaling") } // Create URL & HTTP request. u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/json") @@ -831,14 +842,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } defer resp.Body.Close() // Read body. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return errors.Wrap(err, "reading") } // Handle response based on status code. @@ -862,7 +873,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram // Build request. req, err := http.NewRequest("POST", u.String(), nil) if err != nil { - return err + return errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -870,7 +881,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return err + return errors.Wrap(err, "executing request") } resp.Body.Close() @@ -888,7 +899,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame)) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -896,7 +907,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string // Execute request against the host. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -913,7 +924,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string // Decode response. var rsp getFrameViewsResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Views, nil } @@ -932,7 +943,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("User-Agent", "pilosa/"+Version) @@ -940,7 +951,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -956,7 +967,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v // Decode response object. var rsp getFragmentBlocksResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Blocks, nil } @@ -971,13 +982,13 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s Block: uint64(block), }) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "marshaling") } u := uriPathToURL(c.defaultURI, "/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/protobuf") req.Header.Set("Content-Length", strconv.Itoa(len(buf))) @@ -986,7 +997,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1002,9 +1013,9 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s // Decode response object. var rsp internal.BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "reading") } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "unmarshalling") } return rsp.RowIDs, rsp.ColumnIDs, nil } @@ -1016,13 +1027,13 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Encode request. buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1030,7 +1041,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1044,7 +1055,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b // Decode response object. var rsp postIndexAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Attrs, nil } @@ -1056,13 +1067,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Encode request. buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) if err != nil { - return nil, err + return nil, errors.Wrap(err, "marshaling") } // Build request. req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating request") } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1070,7 +1081,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() @@ -1086,19 +1097,19 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin // Decode response object. var rsp postFrameAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, err + return nil, errors.Wrap(err, "decoding") } return rsp.Attrs, nil } // SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, pb proto.Message) error { +func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return fmt.Errorf("marshaling message: %v", err) } - u := uriPathToURL(ctx.Value("uri").(*URI), "/cluster/message") + u := uriPathToURL(uri, "/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("User-Agent", "pilosa/"+Version) @@ -1126,38 +1137,6 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, pb proto.Message) return nil } -func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI { - clientURI := c.defaultURI - if contextURI, ok := ctx.Value("uri").(*URI); ok { - clientURI = contextURI - } - return clientURI -} - -func (c *InternalHTTPClient) NodeID(uri *URI) (string, error) { - u := uriPathToURL(uri, "/id") - req, err := http.NewRequest("GET", u.String(), nil) - resp, err := c.HTTPClient.Do(req) - if err != nil { - return "", fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return "", fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - return string(body), nil -} - // Bit represents the location of a single bit. type Bit struct { RowID uint64 @@ -1318,8 +1297,8 @@ func uriPathToURL(uri *URI, path string) url.URL { func nodePathToURL(node *Node, path string) url.URL { return url.URL{ - Scheme: node.Scheme, - Host: node.Host, + Scheme: node.URI.Scheme(), + Host: node.URI.HostPort(), Path: path, } } @@ -1336,7 +1315,8 @@ type InternalClient interface { Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) - ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error ImportK(ctx context.Context, index, frame string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error @@ -1353,6 +1333,5 @@ type InternalClient interface { BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, pb proto.Message) error - NodeID(uri *URI) (string, error) + SendMessage(ctx context.Context, uri *URI, pb proto.Message) error } diff --git a/client_test.go b/client_test.go index dfcc3072f..e2b806412 100644 --- a/client_test.go +++ b/client_test.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -36,10 +37,9 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.URI = server[i].HostURI() - server[i].Handler.Cluster = c - server[i].Handler.Cluster.Nodes[i].Host = server[i].Host() - server[i].Handler.Holder = hldr[i].Holder + server[i].Handler.API.Cluster = c + server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() + server[i].Handler.API.Holder = hldr[i].Holder } return server, hldr } @@ -47,7 +47,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { var defaultClient *http.Client func init() { - defaultClient = pilosa.GetHTTPClient(nil) + defaultClient = server.GetHTTPClient(nil) } @@ -64,24 +64,21 @@ func TestClient_MultiNode(t *testing.T) { s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[0].Holder - e.Scheme = cluster.Nodes[0].Scheme - e.Host = cluster.Nodes[0].Host + e.Node = cluster.Nodes[0] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[1].Holder - e.Scheme = cluster.Nodes[1].Scheme - e.Host = cluster.Nodes[1].Host + e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(defaultClient) e.Holder = hldr[2].Holder - e.Scheme = cluster.Nodes[2].Scheme - e.Host = cluster.Nodes[2].Host + e.Node = cluster.Nodes[2] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } @@ -89,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false for _, ownNum := range owns { if ownNum == num { @@ -152,7 +149,7 @@ func TestClient_MultiNode(t *testing.T) { Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), Remote: false, } - result, err := client[0].ExecuteQuery(context.Background(), "i", queryRequest) + result, err := client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } @@ -170,7 +167,7 @@ func TestClient_MultiNode(t *testing.T) { hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) - result, err = client[0].ExecuteQuery(context.Background(), "i", queryRequest) + result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } @@ -190,11 +187,11 @@ func TestClient_MultiNode(t *testing.T) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } - result1, err := client[1].ExecuteQuery(context.Background(), "i", queryRequest) + result1, err := client[1].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } - result2, err := client[2].ExecuteQuery(context.Background(), "i", queryRequest) + result2, err := client[2].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } @@ -220,10 +217,9 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -271,10 +267,9 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -313,17 +308,16 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) if err != nil { t.Fatal(err) } s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -335,14 +329,44 @@ func TestClient_ImportValue(t *testing.T) { t.Fatal(err) } + // Verify Sum. sum, cnt, err := frame.FieldSum(nil, fld.Name) if err != nil { t.Fatal(err) } - - // Verify data. if sum != 50 || cnt != 3 { - t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify Min. + min, cnt, err := frame.FieldMin(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -10 || cnt != 1 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) + } + + // Verify Min with Filter. + filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + if err != nil { + t.Fatal(err) + } + min, cnt, err = frame.FieldMin(filter, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -100 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + } + + // Verify Max. + max, cnt, err := frame.FieldMax(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if max != 40 || cnt != 1 { + t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) } } @@ -358,10 +382,9 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -423,10 +446,10 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -460,10 +483,9 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -489,10 +511,9 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.URI = s.HostURI() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) diff --git a/cluster.go b/cluster.go index d51e1f88f..9dfeac094 100644 --- a/cluster.go +++ b/cluster.go @@ -15,69 +15,105 @@ package pilosa import ( + "context" "encoding/binary" + "fmt" "hash/fnv" + "io/ioutil" + "math/rand" + "net/http" + "os" + "path/filepath" + "sort" "sync" "time" + "golang.org/x/sync/errgroup" + + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" + uuid "github.com/satori/go.uuid" ) const ( // DefaultPartitionN is the default number of partitions in a cluster. DefaultPartitionN = 256 - // DefaultReplicaN is the default number of replicas per partition. - DefaultReplicaN = 1 -) + // ClusterState represents the state returned in the /status endpoint. + ClusterStateStarting = "STARTING" + ClusterStateNormal = "NORMAL" + ClusterStateResizing = "RESIZING" -// NodeState represents node state returned in /status endpoint for a node in the cluster. -const ( - NodeStateUp = "UP" - NodeStateDown = "DOWN" + // NodeState represents the state of a node during startup. + NodeStateLoading = "LOADING" + NodeStateReady = "READY" + + // ResizeJob states. + ResizeJobStateRunning = "RUNNING" + // Final states. + ResizeJobStateDone = "DONE" + ResizeJobStateAborted = "ABORTED" + + ResizeJobActionAdd = "ADD" + ResizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. type Node struct { - Scheme string `json:"scheme"` - Host string `json:"host"` - - mu sync.RWMutex - status *internal.NodeStatus `json:"status"` + ID string `json:"id"` + URI URI `json:"uri"` + IsCoordinator bool `json:"isCoordinator"` } -// Status gets the NodeStatus. -func (n *Node) Status() *internal.NodeStatus { - n.mu.RLock() - defer n.mu.RUnlock() - return n.status +func (n Node) String() string { + return fmt.Sprintf("Node: %s", n.ID) } -// SetStatus sets the NodeStatus. -func (n *Node) SetStatus(s *internal.NodeStatus) { - n.mu.Lock() - n.status = s - n.mu.Unlock() -} - -// SetState sets the Node.status.state. -func (n *Node) SetState(s string) { - n.mu.Lock() - defer n.mu.Unlock() - if n.status == nil { - n.status = &internal.NodeStatus{} +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = EncodeNode(a[i]) } - n.status.State = s + return other } -// URI returns the pilosa.URI corresponding to this node -func (n *Node) URI() (*URI, error) { - uri, err := NewURIFromAddress(n.Host) - if err != nil { - return nil, err +// EncodeNode converts a Node into its internal representation. +func EncodeNode(n *Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: n.URI.Encode(), + IsCoordinator: n.IsCoordinator, + } +} + +// DecodeNodes converts a proto message into a slice of Nodes. +func DecodeNodes(a []*internal.Node) []*Node { + if len(a) == 0 { + return nil + } + other := make([]*Node, len(a)) + for i := range a { + other[i] = DecodeNode(a[i]) + } + return other +} + +// DecodeNode converts a proto message into a Node. +func DecodeNode(node *internal.Node) *Node { + return &Node{ + ID: node.ID, + URI: decodeURI(node.URI), + IsCoordinator: node.IsCoordinator, + } +} + +func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { + return &NodeEvent{ + Event: NodeEventType(ne.Event), + Node: DecodeNode(ne.Node), } - uri.SetScheme(n.Scheme) - return uri, nil } // Nodes represents a list of nodes. @@ -93,10 +129,10 @@ func (a Nodes) Contains(n *Node) bool { return false } -// ContainsHost returns true if host matches one of the node's host. -func (a Nodes) ContainsHost(host string) bool { +// ContainsID returns true if host matches one of the node's id. +func (a Nodes) ContainsID(id string) bool { for _, n := range a { - if n.Host == host { + if n.ID == id { return true } } @@ -114,24 +150,44 @@ func (a Nodes) Filter(n *Node) []*Node { return other } -// FilterHost returns a new list of nodes with host removed. -func (a Nodes) FilterHost(host string) []*Node { +// FilterID returns a new list of nodes with ID removed. +func (a Nodes) FilterID(id string) []*Node { other := make([]*Node, 0, len(a)) for _, node := range a { - if node.Host != host { + if node.ID != id { other = append(other, node) } } return other } -// Hosts returns a list of all hostnames. -func (a Nodes) Hosts() []string { - hosts := make([]string, len(a)) - for i, n := range a { - hosts[i] = n.Host +// FilterURI returns a new list of nodes with URI removed. +func (a Nodes) FilterURI(uri URI) []*Node { + other := make([]*Node, 0, len(a)) + for _, node := range a { + if node.URI != uri { + other = append(other, node) + } } - return hosts + return other +} + +// IDs returns a list of all node IDs. +func (a Nodes) IDs() []string { + ids := make([]string, len(a)) + for i, n := range a { + ids[i] = n.ID + } + return ids +} + +// URIs returns a list of all uris. +func (a Nodes) URIs() []URI { + uris := make([]URI, len(a)) + for i, n := range a { + uris[i] = n.URI + } + return uris } // Clone returns a shallow copy of nodes. @@ -141,10 +197,26 @@ func (a Nodes) Clone() []*Node { return other } +// byID implements sort.Interface for []Node based on +// the ID field. +type byID []*Node + +func (h byID) Len() int { return len(h) } +func (h byID) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h byID) Less(i, j int) bool { return h[i].ID < h[j].ID } + +// nodeAction represents a node that is joining or leaving the cluster. +type nodeAction struct { + node *Node + action string +} + // Cluster represents a collection of nodes. type Cluster struct { - Nodes []*Node - NodeSet NodeSet + ID string + Node *Node + Nodes []*Node // TODO phase this out? + MemberSet MemberSet // Hashing algorithm used to assign partitions to nodes. Hasher Hasher @@ -160,71 +232,575 @@ type Cluster struct { // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int + + // EventReceiver receives NodeEvents pertaining to node membership. + EventReceiver EventReceiver + + // Data directory path. + Path string + Topology *Topology + + // Required for cluster Resize. + Static bool // Static is primarily used for testing in a non-gossip environment. + state string + Coordinator string + Holder *Holder + Broadcaster Broadcaster + + joiningLeavingNodes chan nodeAction + + // joining is held open until this node + // receives ClusterStatus from the coordinator. + joining chan struct{} + joined bool + + mu sync.RWMutex + jobs map[int64]*ResizeJob + currentJob *ResizeJob + + // Close management + wg sync.WaitGroup + closing chan struct{} + + Logger Logger + + // + RemoteClient *http.Client } // NewCluster returns a new instance of Cluster with defaults. func NewCluster() *Cluster { return &Cluster{ - Hasher: &jmphasher{}, - PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, + Hasher: &jmphasher{}, + PartitionN: DefaultPartitionN, + ReplicaN: 1, + EventReceiver: NopEventReceiver, + + joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel + jobs: make(map[int64]*ResizeJob), + closing: make(chan struct{}), + joining: make(chan struct{}), + + Logger: NopLogger, } } -// NodeSetHosts returns the list of host strings for NodeSet members. -func (c *Cluster) NodeSetHosts() []string { - if c.NodeSet == nil { - return []string{} - } - a := make([]string, 0, len(c.NodeSet.Nodes())) - for _, m := range c.NodeSet.Nodes() { - a = append(a, m.Host) - } - return a +// Coordinator returns the coordinator node. +func (c *Cluster) CoordinatorNode() *Node { + return c.nodeByID(c.Coordinator) } -// NodeStates returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. -func (c *Cluster) NodeStates() map[string]string { - h := make(map[string]string) - for _, n := range c.Nodes { - h[n.Host] = NodeStateDown +// IsCoordinator is true if this node is the coordinator. +func (c *Cluster) IsCoordinator() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.isCoordinator() +} + +func (c *Cluster) isCoordinator() bool { + return c.Coordinator == c.Node.ID +} + +// SetCoordinator tells the current node to become the +// Coordinator. In response to this, the current node +// will consider itself coordinator and update the other +// nodes with its version of Cluster.Status. +func (c *Cluster) SetCoordinator(n *Node) error { + c.mu.Lock() + // Verify that the new Coordinator value matches + // this node. + if c.Node.ID != n.ID { + c.mu.Unlock() + return fmt.Errorf("coordinator node does not match this node") } - // we are assuming that NodeSetHosts is a subset of c.Nodes - for _, m := range c.NodeSetHosts() { - if _, ok := h[m]; ok { - h[m] = NodeStateUp + + // Update IsCoordinator on all nodes (locally). + _ = c.updateCoordinator(n) + c.mu.Unlock() + // Send the update coordinator message to all nodes. + err := c.Broadcaster.SendSync( + &internal.UpdateCoordinatorMessage{ + New: EncodeNode(n), + }) + if err != nil { + return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) + } + + // Broadcast cluster status. + return c.Broadcaster.SendSync(c.Status()) +} + +// UpdateCoordinator updates this nodes Coordinator value as well as +// changing the corresponding node's IsCoordinator value +// to true, and sets all other nodes to false. Returns true if the value +// changed. +func (c *Cluster) UpdateCoordinator(n *Node) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.updateCoordinator(n) +} + +func (c *Cluster) updateCoordinator(n *Node) bool { + var changed bool + if c.Coordinator != n.ID { + c.Coordinator = n.ID + changed = true + } + for _, node := range c.Nodes { + if node.ID == n.ID { + node.IsCoordinator = true + } else { + node.IsCoordinator = false } } - return h + return changed } +// AddNode adds a node to the Cluster and updates and saves the +// new topology. +func (c *Cluster) AddNode(node *Node) error { + c.Logger.Printf("add node %s to cluster on %s", node, c.Node) + + // If the node being added is the coordinator, set it for this node. + if node.IsCoordinator { + c.Coordinator = node.ID + } + + // add to cluster + if !c.addNodeBasicSorted(node) { + return nil + } + + // add to topology + if c.Topology == nil { + return fmt.Errorf("Cluster.Topology is nil") + } + if !c.Topology.AddID(node.ID) { + return nil + } + + // save topology + return c.saveTopology() +} + +// RemoveNode removes a node from the Cluster and updates and saves the +// new topology. +func (c *Cluster) RemoveNode(node *Node) error { + // remove from cluster + if !c.removeNodeBasicSorted(node) { + return nil + } + + // remove from topology + if c.Topology == nil { + return fmt.Errorf("Cluster.Topology is nil") + } + if !c.Topology.RemoveID(node.ID) { + return nil + } + + // save topology + return c.saveTopology() +} + +// NodeIDs returns the list of IDs in the cluster. +func (c *Cluster) NodeIDs() []string { + return Nodes(c.Nodes).IDs() +} + +func (c *Cluster) setID(id string) { + // Don't overwrite ClusterID. + if c.ID != "" { + return + } + c.ID = id + + // Make sure the Topology is updated. + c.Topology.ClusterID = c.ID +} + +func (c *Cluster) State() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.state +} + +func (c *Cluster) SetState(state string) { + c.mu.Lock() + c.setState(state) + c.mu.Unlock() +} + +func (c *Cluster) setState(state string) { + // Ignore cases where the state hasn't changed. + if state == c.state { + return + } + + c.Logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) + + var doCleanup bool + + switch state { + case ClusterStateNormal: + // If state is RESIZING -> NORMAL then run cleanup. + if c.state == ClusterStateResizing { + doCleanup = true + } + } + + c.state = state + + // TODO: consider NOT running cleanup on an active node that has + // been removed. + // It's safe to do a cleanup after state changes back to normal. + if doCleanup { + var cleaner HolderCleaner + cleaner.Node = c.Node + cleaner.Holder = c.Holder + cleaner.Cluster = c + cleaner.Closing = c.closing + + // Clean holder. + if err := cleaner.CleanHolder(); err != nil { + c.Logger.Printf("holder clean error: err=%s", err) + } + } +} + +func (c *Cluster) SetNodeState(state string) error { + if c.IsCoordinator() { + return c.ReceiveNodeState(c.Node.ID, state) + } + + // Send node state to coordinator. + ns := &internal.NodeStateMessage{ + NodeID: c.Node.ID, + State: state, + } + + c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) + if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { + return fmt.Errorf("sending node state error: err=%s", err) + } + + return nil +} + +// ReceiveNodeState sets node state in Topology in order for the +// Coordinator to keep track of, during startup, which nodes have +// finished opening their Holder. +func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { + if !c.IsCoordinator() { + return nil + } + + // This method is really only useful during initial startup. + if c.State() != ClusterStateStarting { + return nil + } + + c.Topology.nodeStates[nodeID] = state + c.Logger.Printf("received state %s (%s)", state, nodeID) + + // Set cluster state to NORMAL. + if c.haveTopologyAgreement() && c.allNodesReady() { + return c.setStateAndBroadcast(ClusterStateNormal) + } + + return nil +} + +// localNode is not being used. +//func (c *Cluster) localNode() *Node { +// return c.NodeByURI(c.URI) +//} + // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ - Nodes: encodeClusterStatus(c.Nodes), + ClusterID: c.ID, + State: c.state, + Nodes: EncodeNodes(c.Nodes), } } -// encodeClusterStatus converts a into its internal representation. -func encodeClusterStatus(a []*Node) []*internal.NodeStatus { - other := make([]*internal.NodeStatus, len(a)) - for i := range a { - other[i] = a[i].Status() - } - return other +func (c *Cluster) NodeByID(id string) *Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.nodeByID(id) } -// NodeByHost returns a node reference by host. -func (c *Cluster) NodeByHost(host string) *Node { +// nodeByID returns a node reference by ID. +func (c *Cluster) nodeByID(id string) *Node { for _, n := range c.Nodes { - if n.Host == host { + if n.ID == id { return n } } return nil } +// nodeByURI returns a node reference by node URI. +func (c *Cluster) nodeByURI(uri URI) *Node { + for _, n := range c.Nodes { + if n.URI == uri { + return n + } + } + return nil +} + +// nodePositionByID returns the position of the node in slice c.Nodes. +func (c *Cluster) nodePositionByID(nodeID string) int { + for i, n := range c.Nodes { + if n.ID == nodeID { + return i + } + } + return -1 +} + +// addNodeBasicSorted adds a node to the cluster, sorted by id. +// Returns a pointer to the node and true if the node was added. +func (c *Cluster) addNodeBasicSorted(node *Node) bool { + n := c.nodeByID(node.ID) + if n != nil { + return false + } + + c.Nodes = append(c.Nodes, node) + + // All hosts must be merged in the same order on all nodes in the cluster. + sort.Sort(byID(c.Nodes)) + + return true +} + +// removeNodeBasicSorted removes a node from the cluster, maintaining +// the sort order. Returns true if the node was removed. +func (c *Cluster) removeNodeBasicSorted(node *Node) bool { + i := c.nodePositionByID(node.ID) + if i < 0 { + return false + } + + copy(c.Nodes[i:], c.Nodes[i+1:]) + c.Nodes[len(c.Nodes)-1] = nil + c.Nodes = c.Nodes[:len(c.Nodes)-1] + + return true +} + +// frag is a struct of basic fragment information. +type frag struct { + frame string + view string + slice uint64 +} + +func fragsDiff(a, b []frag) []frag { + m := make(map[frag]uint64) + + for _, y := range b { + m[y]++ + } + + var ret []frag + for _, x := range a { + if m[x] > 0 { + m[x]-- + continue + } + ret = append(ret, x) + } + + return ret +} + +type fragsByHost map[string][]frag + +func (a fragsByHost) add(b fragsByHost) fragsByHost { + for k, v := range b { + for _, vv := range v { + a[k] = append(a[k], vv) + } + } + return a +} + +type viewsByFrame map[string][]string + +func (a viewsByFrame) addView(frame, view string) { + a[frame] = append(a[frame], view) +} + +func (c *Cluster) fragsByHost(idx *Index) fragsByHost { + // frameViews is a map of frame to slice of views. + frameViews := make(viewsByFrame) + inverseFrameViews := make(viewsByFrame) + + for _, frame := range idx.Frames() { + for _, view := range frame.Views() { + if IsInverseView(view.Name()) { + inverseFrameViews.addView(frame.Name(), view.Name()) + } else { + frameViews.addView(frame.Name(), view.Name()) + } + } + } + + std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) + inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews) + return std.add(inv) +} + +// fragCombos returns a map (by uri) of lists of fragments for a given index +// by creating every combination of frame/view specified in `frameViews` up to maxSlice. +func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { + t := make(fragsByHost) + for i := uint64(0); i <= maxSlice; i++ { + nodes := c.SliceNodes(idx, i) + for _, n := range nodes { + // for each frame/view combination: + for frame, views := range frameViews { + for _, view := range views { + t[n.ID] = append(t[n.ID], frag{frame, view, i}) + } + } + } + } + return t +} + +// diff compares c with another cluster and determines if a node is being +// added or removed. An error is returned for any case other than where +// exactly one node is added or removed. +func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) { + lenFrom := len(c.Nodes) + lenTo := len(other.Nodes) + // Determine if a node is being added or removed. + if lenFrom == lenTo { + return "", "", errors.New("clusters are the same size") + } + if lenFrom < lenTo { + // Adding a node. + if lenTo-lenFrom > 1 { + return "", "", errors.New("adding more than one node at a time is not supported") + } + action = ResizeJobActionAdd + // Determine the node ID that is being added. + for _, n := range other.Nodes { + if c.nodeByID(n.ID) == nil { + nodeID = n.ID + break + } + } + } else if lenFrom > lenTo { + // Removing a node. + if lenFrom-lenTo > 1 { + return "", "", errors.New("removing more than one node at a time is not supported") + } + action = ResizeJobActionRemove + // Determine the node ID that is being removed. + for _, n := range c.Nodes { + if other.nodeByID(n.ID) == nil { + nodeID = n.ID + break + } + } + } + return action, nodeID, nil +} + +// fragSources returns a list of ResizeSources - for each node in the `to` cluster - +// required to move from cluster `c` to cluster `to`. +func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { + m := make(map[string][]*internal.ResizeSource) + + // Determine if a node is being added or removed. + action, diffNodeID, err := c.diff(to) + if err != nil { + return nil, errors.Wrap(err, "diffing") + } + + // Initialize the map with all the nodes in `to`. + for _, n := range to.Nodes { + m[n.ID] = nil + } + + // If a node is being added, the source can be confined to the + // primary fragments (i.e. no need to use replicas as source data). + // In this case, source fragments can be based on a cluster with + // replica = 1. + // If a node is being removed, however, then it will most likely + // require that a replica fragment be the source data. + srcCluster := c + if action == ResizeJobActionAdd && c.ReplicaN > 1 { + srcCluster = NewCluster() + srcCluster.Nodes = Nodes(c.Nodes).Clone() + srcCluster.Hasher = c.Hasher + srcCluster.PartitionN = c.PartitionN + srcCluster.ReplicaN = 1 + } + + // Represents the fragment location for the from/to clusters. + fFrags := c.fragsByHost(idx) + tFrags := to.fragsByHost(idx) + + // srcFrags is the frag map based on a source cluster of replica = 1. + srcFrags := srcCluster.fragsByHost(idx) + + // srcNodesByFrag is the inverse representation of srcFrags. + srcNodesByFrag := make(map[frag]string) + for nodeID, frags := range srcFrags { + // If a node is being removed, don't consider it as a source. + if action == ResizeJobActionRemove && nodeID == diffNodeID { + continue + } + for _, frag := range frags { + srcNodesByFrag[frag] = nodeID + } + } + + // Get the frag diff for each nodeID. + diffs := make(fragsByHost) + for nodeID, frags := range tFrags { + if _, ok := fFrags[nodeID]; ok { + diffs[nodeID] = fragsDiff(frags, fFrags[nodeID]) + } else { + diffs[nodeID] = frags + } + } + + // Get the ResizeSource for each diff. + for nodeID, diff := range diffs { + m[nodeID] = []*internal.ResizeSource{} + for _, frag := range diff { + // If there is no valid source node ID for a fragment, + // it likely means that the replica factor was not + // high enough for the remaining nodes to contain + // the fragment. + srcNodeID, ok := srcNodesByFrag[frag] + if !ok { + return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") + } + + src := &internal.ResizeSource{ + Node: EncodeNode(c.nodeByID(srcNodeID)), + Index: idx.Name(), + Frame: frag.frame, + View: frag.view, + Slice: frag.slice, + } + + m[nodeID] = append(m[nodeID], src) + } + } + + return m, nil +} + // Partition returns the partition that a slice belongs to. func (c *Cluster) Partition(index string, slice uint64) int { var buf [8]byte @@ -237,14 +813,14 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// FragmentNodes returns a list of nodes that own a fragment. -func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { +// SliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { return c.PartitionNodes(c.Partition(index, slice)) } -// OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(host string, index string, slice uint64) bool { - return Nodes(c.FragmentNodes(index, slice)).ContainsHost(host) +// OwnsSlice returns true if a host owns a fragment. +func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) } // PartitionNodes returns a list of nodes that own a partition. @@ -270,20 +846,36 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices find the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, host string) []uint64 { +// OwnsSlices finds the set of slices owned by the node per Index +func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { p := c.Partition(index, i) // Determine primary owner node. nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].Host == host { + if c.Nodes[nodeIndex].URI == uri { slices = append(slices, i) } } return slices } +// ContainsSlices is like OwnsSlices, but it includes replicas. +func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { + var slices []uint64 + for i := uint64(0); i <= maxSlice; i++ { + p := c.Partition(index, i) + // Determine the nodes for partition. + nodes := c.PartitionNodes(p) + for _, n := range nodes { + if n.ID == node.ID { + slices = append(slices, i) + } + } + } + return slices +} + // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). @@ -306,3 +898,972 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } + +func (c *Cluster) Open() error { + // Cluster always comes up in state STARTING until cluster membership is determined. + c.state = ClusterStateStarting + + // Load topology file if it exists. + if err := c.loadTopology(); err != nil { + return errors.Wrap(err, "loading topology") + } + + c.ID = c.Topology.ClusterID + + // Only the coordinator needs to consider the .topology file. + if c.IsCoordinator() { + err := c.considerTopology() + if err != nil { + return fmt.Errorf("considerTopology: %v", err) + } + } + + // Add the local node to the cluster. + err := c.AddNode(c.Node) + if err != nil { + return errors.Wrap(err, "adding local node") + } + + // Start the EventReceiver. + if err := c.EventReceiver.Start(c); err != nil { + return fmt.Errorf("starting EventReceiver: %v", err) + } + + // Open MemberSet communication. + if err := c.MemberSet.Open(c.Node); err != nil { + return fmt.Errorf("opening MemberSet: %v", err) + } + + // If not coordinator then wait for ClusterStatus from coordinator. + if !c.IsCoordinator() { + // In the case where a node has been restarted and memberlist has + // not had enough time to determine the node went down/up, then + // the coorninator needs to be alerted that this node is back up + // (and now in a state of STARTING) so that it can be put to the correct + // cluster state. + // TODO: Because the normal code path already sends a NodeJoin event (via + // memberlist), this it a bit redundant in most cases. Perhaps determine + // that the node has been restarted and don't do this step. + msg := &internal.NodeEventMessage{ + Event: uint32(NodeJoin), + Node: EncodeNode(c.Node), + } + if err := c.Broadcaster.SendAsync(msg); err != nil { + return fmt.Errorf("sending restart NodeJoin: %v", err) + } + + c.Logger.Printf("wait for joining to complete") + <-c.joining + c.Logger.Printf("joining has completed") + } + + return nil +} + +func (c *Cluster) Close() error { + // Notify goroutines of closing and wait for completion. + close(c.closing) + c.wg.Wait() + + return nil +} + +func (c *Cluster) markAsJoined() { + c.Logger.Printf("mark node as joined (received coordinator update)") + if !c.joined { + c.joined = true + close(c.joining) + } +} + +func (c *Cluster) needTopologyAgreement() bool { + return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) +} + +func (c *Cluster) haveTopologyAgreement() bool { + if c.Static { + return true + } + return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) +} + +func (c *Cluster) allNodesReady() bool { + if c.Static { + return true + } + for _, uri := range c.Topology.NodeIDs { + if c.Topology.nodeStates[uri] != NodeStateReady { + return false + } + } + return true +} + +func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { + j, err := c.generateResizeJob(nodeAction) + if err != nil { + c.Logger.Printf("generateResizeJob error: err=%s", err) + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + } + return errors.Wrap(err, "setting state") + } + + // j.Run() runs in a goroutine because in the case where the + // job requires no action, it immediately writes to the j.result + // channel, which is not consumed until the code below. + var eg errgroup.Group + eg.Go(func() error { + return j.Run() + }) + + // Wait for the ResizeJob to finish or be aborted. + c.Logger.Printf("wait for jobResult") + jobResult := <-j.result + + // Make sure j.Run() didn't return an error. + if eg.Wait() != nil { + return errors.Wrap(err, "running job") + } + + c.Logger.Printf("received jobResult: %s", jobResult) + switch jobResult { + case ResizeJobStateDone: + if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + return errors.Wrap(err, "completing finished job") + } + // Add/remove uri to/from the cluster. + if j.action == ResizeJobActionRemove { + return c.RemoveNode(nodeAction.node) + } else if j.action == ResizeJobActionAdd { + return c.AddNode(nodeAction.node) + } + case ResizeJobStateAborted: + if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + return errors.Wrap(err, "completing aborted job") + } + } + return nil +} + +func (c *Cluster) setStateAndBroadcast(state string) error { + c.SetState(state) + // Broadcast cluster status changes to the cluster. + c.Logger.Printf("broadcasting ClusterStatus: %s", state) + return c.Broadcaster.SendSync(c.Status()) +} + +func (c *Cluster) sendTo(node *Node, msg proto.Message) error { + if err := c.Broadcaster.SendTo(node, msg); err != nil { + return errors.Wrap(err, "sending") + } + return nil +} + +// ListenForJoins handles cluster-resize events. +func (c *Cluster) ListenForJoins() { + c.wg.Add(1) + go func() { defer c.wg.Done(); c.listenForJoins() }() +} + +func (c *Cluster) listenForJoins() { + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + + var setNormal bool + + for { + + // Handle all pending joins before changing state back to NORMAL. + select { + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + default: + } + + // Only change state to NORMAL if we have successfully added at least one host. + if setNormal { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + } + } +} + +// generateResizeJob creates a new ResizeJob based on the new node being +// added/removed. It also saves a reference to the ResizeJob in the `jobs` map +// for future lookup by JobID. +func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { + c.Logger.Printf("generateResizeJob: %v", nodeAction) + c.mu.Lock() + defer c.mu.Unlock() + + j, err := c.generateResizeJobByAction(nodeAction) + if err != nil { + return nil, errors.Wrap(err, "generating job") + } + c.Logger.Printf("generated ResizeJob: %d", j.ID) + + // Save job in jobs map for future reference. + c.jobs[j.ID] = j + + // Set job as currentJob. + if c.currentJob != nil { + return nil, fmt.Errorf("there is currently a resize job running") + } + c.currentJob = j + + return j, nil +} + +// generateResizeJobByAction returns a ResizeJob with instructions based on +// the difference between Cluster and a new Cluster with/without uri. +// Broadcaster is associated to the ResizeJob here for use in broadcasting +// the resize instructions to other nodes in the cluster. +func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { + j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) + j.Broadcaster = c.Broadcaster + + // toCluster is a clone of Cluster with the new node added/removed for comparison. + toCluster := NewCluster() + toCluster.Nodes = Nodes(c.Nodes).Clone() + toCluster.Hasher = c.Hasher + toCluster.PartitionN = c.PartitionN + toCluster.ReplicaN = c.ReplicaN + if nodeAction.action == ResizeJobActionRemove { + toCluster.removeNodeBasicSorted(nodeAction.node) + } else if nodeAction.action == ResizeJobActionAdd { + toCluster.addNodeBasicSorted(nodeAction.node) + } + + // multiIndex is a map of sources initialized with all the nodes in toCluster. + multiIndex := make(map[string][]*internal.ResizeSource) + + for _, n := range toCluster.Nodes { + multiIndex[n.ID] = nil + } + + // Add to multiIndex the instructions for each index. + for _, idx := range c.Holder.Indexes() { + fragSources, err := c.fragSources(toCluster, idx) + if err != nil { + return nil, errors.Wrap(err, "getting sources") + } + + for id, sources := range fragSources { + for _, src := range sources { + multiIndex[id] = append(multiIndex[id], src) + } + } + } + + for id, sources := range multiIndex { + // If a host doesn't need to request data, mark it as complete. + if len(sources) == 0 { + j.IDs[id] = true + continue + } + instr := &internal.ResizeInstruction{ + JobID: j.ID, + Node: EncodeNode(toCluster.nodeByID(id)), + Coordinator: EncodeNode(c.CoordinatorNode()), + Sources: sources, + Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. + ClusterStatus: c.Status(), + } + j.Instructions = append(j.Instructions, instr) + } + + return j, nil +} + +// CompleteCurrentJob sets the state of the current ResizeJob +// then removes the pointer to currentJob. +func (c *Cluster) CompleteCurrentJob(state string) error { + c.mu.Lock() + defer c.mu.Unlock() + if !c.isCoordinator() { + return ErrNodeNotCoordinator + } + if c.currentJob == nil { + return ErrResizeNotRunning + } + c.currentJob.SetState(state) + c.currentJob = nil + return nil +} + +// FollowResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + c.Logger.Printf("follow resize instruction on %s", c.Node.ID) + // Make sure the cluster status on this node agrees with the Coordinator + // before attempting a resize. + if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + return errors.Wrap(err, "merging cluster status") + } + + c.Logger.Printf("MergeClusterStatus done, start goroutine") + + // The actual resizing runs in a goroutine because we don't want to block + // the distribution of other ResizeInstructions to the rest of the cluster. + go func() { + + // Make sure the holder has opened. + <-c.Holder.opened + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Node: instr.Node, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + + // Sync the schema received in the resize instruction. + c.Logger.Printf("Holder ApplySchema") + if err := c.Holder.ApplySchema(instr.Schema); err != nil { + return errors.Wrap(err, "applying schema") + } + + // Create a client for calling remote nodes. + client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions + + // Request each source file in ResizeSources. + for _, src := range instr.Sources { + c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + + srcURI := decodeURI(src.Node.URI) + + // Retrieve frame. + f := c.Holder.Frame(src.Index, src.Frame) + if f == nil { + return ErrFrameNotFound + } + + // Create view. + v, err := f.CreateViewIfNotExists(src.View) + if err != nil { + return errors.Wrap(err, "creating view") + } + + // Create the local fragment. + frag, err := v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + + // Stream slice from remote node. + c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + if err != nil { + // For now it is an acceptable error if the fragment is not found + // on the remote node. This occurs when a slice has been skipped and + // therefore doesn't contain data. The coordinator correctly determined + // the resize instruction to retrieve the slice, but it doesn't have data. + // TODO: figure out a way to distinguish from "fragment not found" errors + // which are true errors and which simply mean the fragment doesn't have data. + if err == ErrFragmentNotFound { + return nil + } + return errors.Wrap(err, "retrieving slice") + } else if rd == nil { + return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) + } + + // Write to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + return errors.Wrap(err, "copying remote slice") + } + } + return nil + }(); err != nil { + complete.Error = err.Error() + } + + if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { + c.Logger.Printf("sending resizeInstructionComplete error: err=%s", err) + } + }() + return nil +} + +func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { + + j := c.Job(complete.JobID) + + // Abort the job if an error exists in the complete object. + if complete.Error != "" { + j.result <- ResizeJobStateAborted + return errors.New(complete.Error) + } + + j.mu.Lock() + defer j.mu.Unlock() + + if j.isComplete() { + return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + } + + // Mark host complete. + j.IDs[complete.Node.ID] = true + + if !j.nodesArePending() { + j.result <- ResizeJobStateDone + } + + return nil +} + +// Job returns a ResizeJob by id. +func (c *Cluster) Job(id int64) *ResizeJob { + c.mu.RLock() + defer c.mu.RUnlock() + return c.job(id) +} + +func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } + +type ResizeJob struct { + ID int64 + IDs map[string]bool + Instructions []*internal.ResizeInstruction + Broadcaster Broadcaster + + action string + result chan string + + mu sync.RWMutex + state string + + Logger Logger +} + +// NewResizeJob returns a new instance of ResizeJob. +func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { + + // Build a map of uris to track their resize status. + // The value for a node will be set to true after that node + // has indicated that it has completed all resize instructions. + ids := make(map[string]bool) + + if action == ResizeJobActionRemove { + for _, n := range existingNodes { + // Exclude the removed node from the map. + if n.ID == node.ID { + continue + } + ids[n.ID] = false + } + } else if action == ResizeJobActionAdd { + for _, n := range existingNodes { + ids[n.ID] = false + } + // Include the added node in the map for tracking. + ids[node.ID] = false + } + + return &ResizeJob{ + ID: rand.Int63(), + IDs: ids, + action: action, + result: make(chan string), + Logger: NopLogger, + } +} + +func (j *ResizeJob) State() string { + j.mu.RLock() + defer j.mu.RUnlock() + return j.state +} + +func (j *ResizeJob) SetState(state string) { + j.mu.Lock() + j.setState(state) + j.mu.Unlock() +} + +func (j *ResizeJob) setState(state string) { + if j.state == "" || j.state == ResizeJobStateRunning { + j.state = state + } +} + +// Run distributes ResizeInstructions. +func (j *ResizeJob) Run() error { + j.Logger.Printf("run ResizeJob") + // Set job state to RUNNING. + j.SetState(ResizeJobStateRunning) + + // Job can be considered done in the case where it doesn't require any action. + if !j.nodesArePending() { + j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") + j.result <- ResizeJobStateDone + return nil + } + + j.Logger.Printf("distribute tasks for ResizeJob") + err := j.distributeResizeInstructions() + if err != nil { + j.result <- ResizeJobStateAborted + return errors.Wrap(err, "distributing instructions") + } + return nil +} + +// isComplete return true if the job is any one of several completion states. +func (j *ResizeJob) isComplete() bool { + switch j.state { + case ResizeJobStateDone, ResizeJobStateAborted: + return true + default: + return false + } +} + +// nodesArePending returns true if any node is still working on the resize. +func (j *ResizeJob) nodesArePending() bool { + for _, complete := range j.IDs { + if !complete { + return true + } + } + return false +} + +func (j *ResizeJob) distributeResizeInstructions() error { + j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) + // Loop through the ResizeInstructions in ResizeJob and send to each host. + for _, instr := range j.Instructions { + // Because the node may not be in the cluster yet, create + // a dummy node object to use in the SendTo() method. + node := &Node{ + ID: instr.Node.ID, + URI: decodeURI(instr.Node.URI), + } + j.Logger.Printf("send resize instructions: %v", instr) + if err := j.Broadcaster.SendTo(node, instr); err != nil { + return errors.Wrap(err, "sending instruction") + } + } + return nil +} + +type NodeIDs []string + +func (n NodeIDs) Len() int { return len(n) } +func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } +func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] } + +// ContainsID returns true if idi matches one of the nodesets's IDs. +func (n NodeIDs) ContainsID(id string) bool { + for _, nid := range n { + if nid == id { + return true + } + } + return false +} + +// Topology represents the list of hosts in the cluster. +type Topology struct { + mu sync.RWMutex + NodeIDs []string + + ClusterID string + + // nodeStates holds the state of each node according to + // the coordinator. Used during startup and data load. + nodeStates map[string]string +} + +func NewTopology() *Topology { + return &Topology{ + nodeStates: make(map[string]string), + } +} + +// ContainsID returns true if id matches one of the topology's IDs. +func (t *Topology) ContainsID(id string) bool { + t.mu.RLock() + defer t.mu.RUnlock() + return t.containsID(id) +} + +func (t *Topology) containsID(id string) bool { + return NodeIDs(t.NodeIDs).ContainsID(id) +} + +func (t *Topology) positionByID(nodeID string) int { + for i, tid := range t.NodeIDs { + if tid == nodeID { + return i + } + } + return -1 +} + +// AddID adds the node ID to the topology and returns true if added. +func (t *Topology) AddID(nodeID string) bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.containsID(nodeID) { + return false + } + t.NodeIDs = append(t.NodeIDs, nodeID) + + sort.Slice(t.NodeIDs, + func(i, j int) bool { + return t.NodeIDs[i] < t.NodeIDs[j] + }) + + return true +} + +// RemoveID removes the node ID from the topology and returns true if removed. +func (t *Topology) RemoveID(nodeID string) bool { + t.mu.Lock() + defer t.mu.Unlock() + + i := t.positionByID(nodeID) + if i < 0 { + return false + } + + copy(t.NodeIDs[i:], t.NodeIDs[i+1:]) + t.NodeIDs[len(t.NodeIDs)-1] = "" + t.NodeIDs = t.NodeIDs[:len(t.NodeIDs)-1] + + return true +} + +// Encode converts t into its internal representation. +func (t *Topology) Encode() *internal.Topology { + return encodeTopology(t) +} + +// loadTopology reads the topology for the node. +func (c *Cluster) loadTopology() error { + buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) + if os.IsNotExist(err) { + c.Topology = NewTopology() + return nil + } else if err != nil { + return errors.Wrap(err, "reading file") + } + + var pb internal.Topology + if err := proto.Unmarshal(buf, &pb); err != nil { + return errors.Wrap(err, "unmarshalling") + } + top, err := decodeTopology(&pb) + if err != nil { + return errors.Wrap(err, "decoding") + } + c.Topology = top + + return nil +} + +// saveTopology writes the current topology to disk. +func (c *Cluster) saveTopology() error { + + if err := os.MkdirAll(c.Path, 0777); err != nil { + return errors.Wrap(err, "creating directory") + } + + if buf, err := proto.Marshal(encodeTopology(c.Topology)); err != nil { + return errors.Wrap(err, "marshalling") + } else if err := ioutil.WriteFile(filepath.Join(c.Path, ".topology"), buf, 0666); err != nil { + return errors.Wrap(err, "writing file") + } + return nil +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + ClusterID: topology.ClusterID, + NodeIDs: topology.NodeIDs, + } +} + +func decodeTopology(topology *internal.Topology) (*Topology, error) { + if topology == nil { + return nil, nil + } + + t := NewTopology() + t.ClusterID = topology.ClusterID + t.NodeIDs = topology.NodeIDs + sort.Slice(t.NodeIDs, + func(i, j int) bool { + return t.NodeIDs[i] < t.NodeIDs[j] + }) + + return t, nil +} + +func (c *Cluster) considerTopology() error { + // Create ClusterID if one does not already exist. + if c.ID == "" { + u := uuid.NewV4() + c.ID = u.String() + c.Topology.ClusterID = c.ID + } + + if c.Static { + return nil + } + + // If there is no .topology file, it's safe to proceed. + if len(c.Topology.NodeIDs) == 0 { + return nil + } + + // The local node (coordinator) must be in the .topology. + if !c.Topology.ContainsID(c.Node.ID) { + return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.NodeIDs) + } + + // If local node is the only thing in .topology, continue. + //if len(c.Topology.NodeIDs) == 1 { + // return nil + //} + + // Keep the cluster in state "STARTING" until hearing from all nodes. + // Topology contains 2+ hosts. + return nil +} + +// ReceiveEvent represents an implementation of EventHandler. +func (c *Cluster) ReceiveEvent(e *NodeEvent) error { + // Ignore events sent from this node. + if e.Node.ID == c.Node.ID { + return nil + } + + switch e.Event { + case NodeJoin: + c.Logger.Printf("received NodeJoin event: %v", e) + // Ignore the event if this is not the coordinator. + if !c.IsCoordinator() { + return nil + } + return c.nodeJoin(e.Node) + case NodeLeave: + // Automatic nodeLeave is intentionally not implemented. + case NodeUpdate: + // NodeUpdate is intentionally not implemented. + } + + return nil +} + +func (c *Cluster) nodeJoin(node *Node) error { + if c.needTopologyAgreement() { + // A host that is not part of the topology can't be added to the STARTING cluster. + if !c.Topology.ContainsID(node.ID) { + err := fmt.Sprintf("host is not in topology: %s", node.ID) + c.Logger.Printf("%v", err) + return errors.New(err) + } + + if err := c.AddNode(node); err != nil { + return errors.Wrap(err, "adding node for agreement") + } + + // Only change to normal if there is no existing data. Otherwise, + // the coordinator needs to wait to receive READY messages (nodeStates) + // from remote nodes before setting the cluster to state NORMAL. + if ok, err := c.Holder.HasData(); !ok && err == nil { + // If the result of the previous AddNode completed the joining of nodes + // in the topology, then change the state to NORMAL. + if c.haveTopologyAgreement() { + return c.setStateAndBroadcast(ClusterStateNormal) + } + return nil + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") + } + + if c.haveTopologyAgreement() && c.allNodesReady() { + return c.setStateAndBroadcast(ClusterStateNormal) + } else { + // Send the status to the remote node. This lets the remote node + // know that it can proceed with opening its Holder. + return c.sendTo(node, c.Status()) + } + + return nil + } + + // If the cluster already contains the node, just send it the cluster status. + // This is useful in the case where a node is restarted or temporarily leaves + // the cluster. + if node := c.nodeByID(node.ID); node != nil { + return c.sendTo(node, c.Status()) + } + + // If the holder does not yet contain data, go ahead and add the node. + if ok, err := c.Holder.HasData(); !ok && err == nil { + if err := c.AddNode(node); err != nil { + return errors.Wrap(err, "adding node") + } + return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data2") + } + + // If the cluster has data, we need to change to RESIZING and + // kick off the resizing process. + if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { + return errors.Wrap(err, "broadcasting state") + } + c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} + + return nil +} + +// NodeLeave initiates the removal of a node from the cluster. +func (c *Cluster) NodeLeave(node *Node) error { + // Refuse the request if this is not the coordinator. + if !c.IsCoordinator() { + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) + } + + if c.State() != ClusterStateNormal { + return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", ClusterStateNormal, c.State()) + } + + // Ensure that node is in the cluster. + if c.nodeByID(node.ID) == nil { + return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) + } + + // Prevent removing the coordinator node (this node). + if node.ID == c.Node.ID { + return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.") + } + + // See if resize job can be generated + _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) + + if err != nil { + return errors.Wrap(err, "generating job") + } + + return c.nodeLeave(node) +} + +func (c *Cluster) nodeLeave(node *Node) error { + // Get the actual node in the local cluster. + n := c.nodeByID(node.ID) + + // Don't do anything else if the cluster doesn't contain the node. + if n == nil { + return nil + } + + // If the holder does not yet contain data, go ahead and remove the node. + if ok, err := c.Holder.HasData(); !ok && err == nil { + if err := c.RemoveNode(n); err != nil { + return errors.Wrap(err, "removing node") + } + return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") + } + + // If the cluster has data then change state to RESIZING and + // kick off the resizing process. + if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { + return errors.Wrap(err, "broadcasting state") + } + c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} + + return nil +} + +func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { + c.mu.Lock() + defer c.mu.Unlock() + c.Logger.Printf("merge cluster status: %v", cs) + // Ignore status updates from self (coordinator). + if c.isCoordinator() { + return nil + } + + // Set ClusterID. + c.setID(cs.ClusterID) + + officialNodes := DecodeNodes(cs.Nodes) + + // Add all nodes from the coordinator. + for _, node := range officialNodes { + if err := c.AddNode(node); err != nil { + return errors.Wrap(err, "adding node") + } + } + + // Remove any nodes not specified by the coordinator + // except for self. Generate a list to remove first + // so that nodes aren't removed mid-loop. + nodeIDsToRemove := []string{} + for _, node := range c.Nodes { + // Don't remove this node. + if node.ID == c.Node.ID { + continue + } + if Nodes(officialNodes).ContainsID(node.ID) { + continue + } + nodeIDsToRemove = append(nodeIDsToRemove, node.ID) + } + + for _, nodeID := range nodeIDsToRemove { + if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { + return errors.Wrap(err, "removing node") + } + } + + c.setState(cs.State) + + c.markAsJoined() + + return nil +} diff --git a/cluster_internal_test.go b/cluster_internal_test.go new file mode 100644 index 000000000..6b1f6e72b --- /dev/null +++ b/cluster_internal_test.go @@ -0,0 +1,310 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "reflect" + "strings" + "testing" + + "github.com/pilosa/pilosa/internal" +) + +// Ensure that fragCombos creates the correct fragment mapping. +func TestFragCombos(t *testing.T) { + + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + + c := NewCluster() + c.addNodeBasicSorted(node0) + c.addNodeBasicSorted(node1) + + tests := []struct { + idx string + maxSlice uint64 + frameViews viewsByFrame + expected fragsByHost + }{ + { + idx: "i", + maxSlice: uint64(2), + frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, + expected: fragsByHost{ + "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, + "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, + }, + }, + { + idx: "foo", + maxSlice: uint64(3), + frameViews: viewsByFrame{"f": []string{"v0"}}, + expected: fragsByHost{ + "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, + "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, + }, + }, + } + for _, test := range tests { + + actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews) + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("expected: %v, but got: %v", test.expected, actual) + } + + } +} + +// newIndexWithTempPath returns a new instance of Index. +func newIndexWithTempPath(name string) *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := NewIndex(path, name) + if err != nil { + panic(err) + } + return index +} + +// Ensure that fragSources creates the correct fragment mapping. +func TestFragSources(t *testing.T) { + + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + uri2, err := NewURIFromAddress("host2") + if err != nil { + t.Fatal(err) + } + uri3, err := NewURIFromAddress("host3") + if err != nil { + t.Fatal(err) + } + + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + node2 := &Node{ID: "node2", URI: *uri2} + node3 := &Node{ID: "node3", URI: *uri3} + + c1 := NewCluster() + c1.ReplicaN = 1 + c1.addNodeBasicSorted(node0) + c1.addNodeBasicSorted(node1) + + c2 := NewCluster() + c2.ReplicaN = 1 + c2.addNodeBasicSorted(node0) + c2.addNodeBasicSorted(node1) + c2.addNodeBasicSorted(node2) + + c3 := NewCluster() + c3.ReplicaN = 2 + c3.addNodeBasicSorted(node0) + c3.addNodeBasicSorted(node1) + + c4 := NewCluster() + c4.ReplicaN = 2 + c4.addNodeBasicSorted(node0) + c4.addNodeBasicSorted(node1) + c4.addNodeBasicSorted(node2) + + c5 := NewCluster() + c5.ReplicaN = 2 + c5.addNodeBasicSorted(node0) + c5.addNodeBasicSorted(node1) + c5.addNodeBasicSorted(node2) + c5.addNodeBasicSorted(node3) + + idx := newIndexWithTempPath("i") + frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 101, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 1300000, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 2600000, nil) + if err != nil { + t.Fatal(err) + } + _, err = frame.SetBit("standard", 1, 3900000, nil) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + from *Cluster + to *Cluster + idx *Index + expected map[string][]*internal.ResizeSource + err string + }{ + { + from: c1, + to: c2, + idx: idx, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{}, + "node1": []*internal.ResizeSource{}, + "node2": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + }, + }, + err: "", + }, + { + from: c4, + to: c3, + idx: idx, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{ + {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + }, + "node1": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + }, + }, + err: "", + }, + { + from: c5, + to: c4, + idx: idx, + expected: map[string][]*internal.ResizeSource{ + "node0": []*internal.ResizeSource{ + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + }, + "node1": []*internal.ResizeSource{ + {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + }, + "node2": []*internal.ResizeSource{}, + }, + err: "", + }, + { + from: c2, + to: c4, + idx: idx, + expected: nil, + err: "clusters are the same size", + }, + { + from: c1, + to: c5, + idx: idx, + expected: nil, + err: "adding more than one node at a time is not supported", + }, + { + from: c5, + to: c1, + idx: idx, + expected: nil, + err: "removing more than one node at a time is not supported", + }, + } + for _, test := range tests { + + actual, err := (test.from).fragSources(test.to, test.idx) + if test.err != "" { + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("expected error: %s, got: %s", test.err, err.Error()) + } + } else { + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("expected: %v, but got: %v", test.expected, actual) + } + } + } +} + +// Ensure that fragSources creates the correct fragment mapping. +func TestResizeJob(t *testing.T) { + + uri0, err := NewURIFromAddress("host0") + if err != nil { + t.Fatal(err) + } + uri1, err := NewURIFromAddress("host1") + if err != nil { + t.Fatal(err) + } + uri2, err := NewURIFromAddress("host2") + if err != nil { + t.Fatal(err) + } + + node0 := &Node{ID: "node0", URI: *uri0} + node1 := &Node{ID: "node1", URI: *uri1} + node2 := &Node{ID: "node2", URI: *uri2} + + tests := []struct { + existingNodes []*Node + node *Node + action string + expectedIDs map[string]bool + }{ + { + existingNodes: []*Node{node0, node1}, + node: node2, + action: ResizeJobActionAdd, + expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, + }, + { + existingNodes: []*Node{node0, node1, node2}, + node: node2, + action: ResizeJobActionRemove, + expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, + }, + } + for _, test := range tests { + + actual := NewResizeJob(test.existingNodes, test.node, test.action) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual.IDs, test.expectedIDs) { + t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs) + } + } +} diff --git a/cluster_test.go b/cluster_test.go index 8ab1ab461..b09820ac8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -12,38 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( + "bytes" "math/rand" "reflect" "testing" "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" ) // Ensure the cluster can fairly distribute partitions across the nodes. func TestCluster_Owners(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {Host: "serverA:1000"}, - {Host: "serverB:1000"}, - {Host: "serverC:1000"}, + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, }, - Hasher: test.NewModHasher(), + Hasher: NewTestModHasher(), ReplicaN: 2, } // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) { + if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) { + if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } } @@ -51,7 +50,7 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := pilosa.NewCluster() + c := NewCluster() c.PartitionN = partitionN partitionID := c.Partition(index, slice) @@ -84,59 +83,448 @@ func TestHasher(t *testing.T) { {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, } { for i, v := range tt.bucket { - if got := pilosa.NewHasher().Hash(tt.key, i+1); got != v { + if got := NewHasher().Hash(tt.key, i+1); got != v { t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) } } } } -// Ensure that an empty cluster returns a valid (empty) NodeSet -func TestCluster_NodeSetHosts(t *testing.T) { - - c := pilosa.Cluster{} - - if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) { - t.Fatalf("unexpected slice of hosts: %s", h) - } -} - -// Ensure cluster can compare its Nodes and Members -func TestCluster_NodeStates(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {Host: "serverA:1000"}, - {Host: "serverB:1000"}, - {Host: "serverC:1000"}, - }, - NodeSet: &pilosa.StaticNodeSet{}, - } - - err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{ - &pilosa.Node{Host: "serverA:1000"}, - &pilosa.Node{Host: "serverC:1000"}, - &pilosa.Node{Host: "serverD:1000"}, - }) - if err != nil { - t.Fatalf("unexpected gossiper nodes: %s", err) - } - - // Verify a DOWN node is reported, and extraneous nodes are ignored - if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": pilosa.NodeStateUp, - "serverB:1000": pilosa.NodeStateDown, - "serverC:1000": pilosa.NodeStateUp, - }) { - t.Fatalf("unexpected node state: %s", spew.Sdump(a)) - } -} - -// Ensure OwnsSlices can find the actual slice list for node and index +// Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { - c := test.NewCluster(5) - slices := c.OwnsSlices("test", 10, "host2") + c := NewTestCluster(5) + slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) } } + +// Ensure ContainsSlices can find the actual slice list for node and index. +func TestCluster_ContainsSlices(t *testing.T) { + c := NewTestCluster(5) + c.ReplicaN = 3 + slices := c.ContainsSlices("test", 10, c.Nodes[2]) + + if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + +func TestCluster_Nodes(t *testing.T) { + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} + + nodes := []*Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterURI", func(t *testing.T) { + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +func TestCluster_Coordinator(t *testing.T) { + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + + c1 := *NewCluster() + c1.Node = node1 + c1.Coordinator = node1.ID + c2 := *NewCluster() + c2.Node = node2 + c2.Coordinator = node1.ID + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.IsCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Node) + } else if c2.IsCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Node) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + + t.Run("AddNode", func(t *testing.T) { + err := c1.AddNode(node1) + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.AddNode(node1) + if err != nil { + t.Fatal(err) + } + err = c1.AddNode(node2) + if err != nil { + t.Fatal(err) + } + + actual := c1.NodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) + } + }) +} + +// Ensure that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + t.Run("Single node, no data", func(t *testing.T) { + tc := NewClusterCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"some-other-host"}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" + err := tc.Open() + if err == nil || err.Error() != expected { + t.Errorf("did not receive expected error: %s", expected) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"node0", "node2"}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: node1" + err := tc.AddNode(false) + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.AddNode(false) + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Bit Data to node0. + if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { + t.Fatal(err) + } + tc.SetBit("i", "f", "standard", 1, 101, nil) + tc.SetBit("i", "f", "standard", 1, 1300000, nil) + + // Add Field Data to node0. + if err := tc.CreateFrame("i", "fields", FrameOptions{ + InverseEnabled: false, + //CacheType: CacheTypeNone, + Fields: []*Field{ + { + Name: "fld0", + Type: FieldTypeInt, + Min: -100, + Max: 100, + }, + }, + }); err != nil { + t.Fatal(err) + } + tc.SetFieldValue("i", "fields", 1, "fld0", -10) + tc.SetFieldValue("i", "fields", 1, "fld0", 10) + tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) + tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) + + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Frame := node0.Holder.Frame("i", "f") + node0View := node0Frame.View("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() + + node0Frame = node0.Holder.Frame("i", "fields") + node0View = node0Frame.View("field_fld0") + node0Fragment = node0View.Fragment(1) + node0ChecksumFld := node0Fragment.Checksum() + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) + node1 := tc.Clusters[1] + + // Ensure that nodes come up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Bits + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node1Frame := node1.Holder.Frame("i", "f") + node1View := node1Frame.View("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) + } + + // Values + // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. + node1Frame = node1.Holder.Frame("i", "fields") + node1View = node1Frame.View("field_fld0") + node1Fragment = node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) { + t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} + +// Ensures that coordinator can be changed. +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { + c := NewTestCluster(2) + + oldNode := c.Nodes[0] + newNode := c.Nodes[1] + + // Update coordinator to the same value. + if c.UpdateCoordinator(oldNode) { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) + } + + // Update coordinator to a new value. + if !c.UpdateCoordinator(newNode) { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) + } + }) +} diff --git a/cmd/import.go b/cmd/import.go index 8692fa318..93fa92ee0 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -60,10 +60,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") - flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame") + flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) diff --git a/cmd/root_test.go b/cmd/root_test.go index 93711a5a1..431d3a66f 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -181,7 +181,6 @@ func TestRootCommand_Config(t *testing.T) { bind = "127.0.0.1:10101" [cluster] - poll-interval = "2m0s" replicas = 2 partitions = 128 hosts = [ diff --git a/cmd/server.go b/cmd/server.go index ffa8c66b3..d906cf189 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -15,18 +15,11 @@ package cmd import ( - "fmt" "io" - "log" - "os" - "os/signal" - "runtime/pprof" - "syscall" - "time" + "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/ctl" "github.com/pilosa/pilosa/server" ) @@ -43,55 +36,13 @@ func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Long: `pilosa server runs Pilosa. It will load existing data from the configured -directory, and start listening client connections +directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr) - if err != nil { - return err + if err := Server.Start(); err != nil { + return errors.Wrap(err, "running server") } - logger := log.New(logOutput, "", log.LstdFlags) - logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) - - // Start CPU profiling. - if Server.CPUProfile != "" { - f, err := os.Create(Server.CPUProfile) - if err != nil { - return fmt.Errorf("create cpu profile: %v", err) - } - defer f.Close() - - fmt.Fprintln(Server.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(Server.CPUTime, func() { - fmt.Fprintln(Server.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. - if err := Server.Run(); err != nil { - return fmt.Errorf("error running server: %v", err) - } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - select { - case sig := <-c: - logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := Server.Close(); err != nil { - return err - } - case <-Server.Done: - logger.Printf("Server closed externally") - } - return nil + return errors.Wrap(Server.Wait(), "waiting on Server") }, } diff --git a/cmd/server_test.go b/cmd/server_test.go index 04436c1c4..abbe8d7a4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/cmd" _ "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa/toml" ) func TestServerHelp(t *testing.T) { @@ -37,23 +37,20 @@ func TestServerHelp(t *testing.T) { func TestServerConfig(t *testing.T) { actualDataDir, err := ioutil.TempDir("", "") failErr(t, err, "making data dir") - profFile, err := ioutil.TempFile("", "") - failErr(t, err, "making temp file") logFile, err := ioutil.TempFile("", "") failErr(t, err, "making log file") tests := []commandTest{ // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" max-writes-per-request = 3000 [cluster] - poll-interval = "45s" - type = "static" + disabled = true replicas = 2 hosts = [ "localhost:19444", @@ -66,8 +63,7 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) - v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) + v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() }, @@ -80,7 +76,7 @@ func TestServerConfig(t *testing.T) { bind = "localhost:0" data-dir = "` + actualDataDir + `" [cluster] - type = "static" + disabled = true hosts = [ "localhost:19444", ] @@ -88,27 +84,23 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) return v.Error() }, }, // TEST 2 { - args: []string{"server", "--log-path", logFile.Name(), "--cluster.type", "static"}, - env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"}, + args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, + env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" data-dir = "` + actualDataDir + `" [cluster] - poll-interval = "2m0s" hosts = [ "localhost:19444", ] [anti-entropy] interval = "11m0s" - [profile] - cpu = "` + profFile.Name() + `" - cpu-time = "35s" [metric] service = "statsd" host = "127.0.0.1:8125" @@ -116,10 +108,7 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Minute*2)) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) - v.Check(cmd.Server.CPUProfile, profFile.Name()) - v.Check(cmd.Server.CPUTime, time.Minute) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11)) v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") @@ -151,6 +140,9 @@ func TestServerConfig(t *testing.T) { case <-cmd.Server.Started: case <-executed: } + if execErr != nil { + t.Fatalf("executing server command: %v", execErr) + } err := cmd.Server.Close() failErr(t, err, "closing pilosa server command") <-executed diff --git a/config.go b/config.go deleted file mode 100644 index cd978c5ca..000000000 --- a/config.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "time" -) - -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - -const ( - // DefaultHost is the default hostname to use. - DefaultHost = "localhost" - - // DefaultPort is the default port use with the hostname. - DefaultPort = "10101" - - // DefaultClusterType sets the node intercommunication method. - DefaultClusterType = ClusterGossip - - // DefaultGossipPort indicates the port to which pilosa should bind for internal state sharing. - DefaultGossipPort = "14000" - - // DefaultMetrics sets the internal metrics to no-op. - DefaultMetrics = "nop" - - // DefaultMaxWritesPerRequest is the default number of writes per request. - DefaultMaxWritesPerRequest = 5000 -) - -// ClusterTypes set of cluster types. -var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterGossip} - -// TLSConfig contains TLS configuration -type TLSConfig struct { - // CertificatePath contains the path to the certificate (.crt or .pem file) - CertificatePath string `toml:"certificate-path"` - // CertificateKeyPath contains the path to the certificate key (.key file) - CertificateKeyPath string `toml:"certificate-key-path"` - // SkipVerify disables verification for self-signed certificates - SkipVerify bool `toml:"skip-verify"` -} - -// Config represents the configuration for the command. -type Config struct { - DataDir string `toml:"data-dir"` - Bind string `toml:"bind"` - // GossipPort DEPRECATED - GossipPort string `toml:"gossip-port"` - // GossipSeed DEPRECATED - GossipSeed string `toml:"gossip-seed"` - - Gossip struct { - Port string `toml:"port"` - Seed string `toml:"seed"` - Key string `toml:"key"` - } `toml:"gossip"` - - Cluster struct { - ReplicaN int `toml:"replicas"` - Type string `toml:"type"` - Hosts []string `toml:"hosts"` - PollInterval Duration `toml:"poll-interval"` - LongQueryTime Duration `toml:"long-query-time"` - } `toml:"cluster"` - - AntiEntropy struct { - Interval Duration `toml:"interval"` - } `toml:"anti-entropy"` - - // Limits the number of mutating commands that can be in a single request to - // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. - MaxWritesPerRequest int `toml:"max-writes-per-request"` - - LogPath string `toml:"log-path"` - - Metric struct { - Service string `toml:"service"` - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` - Diagnostics bool `toml:"diagnostics"` - } `toml:"metric"` - - TLS TLSConfig -} - -// NewConfig returns an instance of Config with default options. -func NewConfig() *Config { - c := &Config{ - Bind: DefaultHost + ":" + DefaultPort, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, - } - c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.Type = DefaultClusterType - c.Cluster.PollInterval = Duration(DefaultPollingInterval) - c.Cluster.Hosts = []string{} - c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) - c.Metric.Service = DefaultMetrics - c.Metric.Diagnostics = true - c.TLS = TLSConfig{} - return c -} - -// Validate that all configuration permutations are compatible with each other. -func (c *Config) Validate() error { - if !StringInSlice(c.Cluster.Type, ClusterTypes) { - return ErrConfigClusterTypeInvalid - } - - if c.Cluster.Type == ClusterGossip { - if len(c.Cluster.Hosts) > 0 { - bindWithDefaults, err := AddressWithDefaults(c.Bind) - if err != nil { - return err - } - if !c.foundHost(bindWithDefaults) { - return ErrConfigHostsMissing - } - } - } - - return nil -} - -func (c *Config) foundHost(host *URI) bool { - for _, clusterHost := range c.Cluster.Hosts { - uri, err := NewURIFromAddress(clusterHost) - if err != nil { - continue - } - if host.Equals(uri) { - return true - } - } - return false -} - -// Duration is a TOML wrapper type for time.Duration. -type Duration time.Duration - -// String returns the string representation of the duration. -func (d Duration) String() string { return time.Duration(d).String() } - -// UnmarshalText parses a TOML value into a duration value. -func (d *Duration) UnmarshalText(text []byte) error { - v, err := time.ParseDuration(string(text)) - if err != nil { - return err - } - - *d = Duration(v) - return nil -} - -// MarshalText writes duration value in text format. -func (d Duration) MarshalText() (text []byte, err error) { - return []byte(d.String()), nil -} - -// MarshalTOML write duration into valid TOML. -func (d Duration) MarshalTOML() ([]byte, error) { - return []byte(d.String()), nil -} diff --git a/ctl/backup.go b/ctl/backup.go index 8e725b41e..54a760695 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -16,11 +16,12 @@ package ctl import ( "context" - "errors" "io" "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // BackupCommand represents a command for backing up a view. @@ -39,7 +40,7 @@ type BackupCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBackupCommand returns a new instance of BackupCommand. @@ -59,26 +60,26 @@ func (cmd *BackupCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Open output file. f, err := os.Create(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "creating file") } defer f.Close() // Begin streaming backup. if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return err + return errors.Wrap(err, "backing up") } // Sync & close file to ensure durability. if err := f.Sync(); err != nil { - return err + return errors.Wrap(err, "syncing") } else if err = f.Close(); err != nil { - return err + return errors.Wrap(err, "closing file") } return nil @@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string { return cmd.Host } -func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 3db4b3b73..a073bc8a8 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -46,14 +46,10 @@ func TestBackupCommand_Run(t *testing.T) { s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.URI = uri - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/bench.go b/ctl/bench.go index 01e07cc14..425894738 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -16,7 +16,6 @@ package ctl import ( "context" - "errors" "fmt" "io" "math/rand" @@ -24,6 +23,8 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // BenchCommand represents a command for benchmarking index operations. @@ -42,7 +43,7 @@ type BenchCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBenchCommand returns a new instance of BenchCommand. @@ -57,7 +58,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } switch cmd.Op { @@ -94,7 +95,7 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalCl Query: fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID), Remote: false, } - if _, err := client.ExecuteQuery(ctx, cmd.Index, queryRequest); err != nil { + if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil { return err } } @@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string { return cmd.Host } -func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/bench_test.go b/ctl/bench_test.go index c3014e6ca..4790ccb44 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pkg/errors" ) func TestBenchCommand_InvalidOption(t *testing.T) { @@ -31,7 +32,7 @@ func TestBenchCommand_InvalidOption(t *testing.T) { cm := NewBenchCommand(stdin, stdout, stderr) err := cm.Run(context.Background()) - if err != pilosa.ErrHostRequired { + if errors.Cause(err) != pilosa.ErrHostRequired { t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err) } diff --git a/ctl/check.go b/ctl/check.go index 12591a0ac..e9589ddf4 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -24,6 +24,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // CheckCommand represents a command for performing consistency checks on data files. @@ -48,17 +49,17 @@ func (cmd *CheckCommand) Run(ctx context.Context) error { switch filepath.Ext(path) { case "": if err := cmd.checkBitmapFile(path); err != nil { - return err + return errors.Wrap(err, "checking bitmap") } case ".cache": if err := cmd.checkCacheFile(path); err != nil { - return err + return errors.Wrap(err, "checking cache") } case ".snapshotting": if err := cmd.checkSnapshotFile(path); err != nil { - return err + return errors.Wrap(err, "checking snapshot") } } } @@ -71,26 +72,26 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error { // Open file handle. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() fi, err := f.Stat() if err != nil { - return err + return errors.Wrap(err, "statting file") } // Memory map the file. data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { - return err + return errors.Wrap(err, "mmapping") } defer syscall.Munmap(data) // Attach the mmap file to the bitmap. bm := roaring.NewSliceBitmap() if err := bm.UnmarshalBinary(data); err != nil { - return err + return errors.Wrap(err, "unmarshalling") } // Perform consistency check. diff --git a/ctl/check_test.go b/ctl/check_test.go index 33a19feb3..56d70f97a 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) { var buf bytes.Buffer io.Copy(&buf, r) - if !strings.HasPrefix(err.Error(), "invalid roaring file") { + if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) } // Todo: need correct roaring file for happy path diff --git a/ctl/common.go b/ctl/common.go index 9bba68657..11042a704 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -18,13 +18,15 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" "github.com/spf13/pflag" ) // CommandWithTLSSupport is the interface for commands which has TLS settings type CommandWithTLSSupport interface { TLSHost() string - TLSConfiguration() pilosa.TLSConfig + TLSConfiguration() server.TLSConfig } // SetTLSConfig creates common TLS flags @@ -41,16 +43,16 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath) if err != nil { - return nil, err + return nil, errors.Wrap(err, "loading keypair") } TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig)) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting internal client") } return client, err } diff --git a/ctl/config.go b/ctl/config.go index 2047941c7..7277f74ea 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -21,12 +21,13 @@ import ( toml "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ConfigCommand represents a command for printing a default config. type ConfigCommand struct { *pilosa.CmdIO - Config *pilosa.Config + Config *server.Config } // NewConfigCommand returns a new instance of ConfigCommand. diff --git a/ctl/config_test.go b/ctl/config_test.go index a3bd035ac..ca08c273b 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -17,11 +17,12 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" "testing" + + "github.com/pilosa/pilosa/server" ) func TestConfigCommand_Run(t *testing.T) { @@ -29,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) { stdin := bytes.NewReader(rder) r, w, _ := os.Pipe() cm := NewConfigCommand(stdin, w, os.Stderr) - cm.Config = pilosa.NewConfig() + cm.Config = server.NewConfig() err := cm.Run(context.Background()) w.Close() @@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) { if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { - t.Fatalf("Unexpected config: %s", buf.String()) + } else if !strings.Contains(buf.String(), ":10101") { + t.Fatalf("Unexpected config: \n%s", buf.String()) } } diff --git a/ctl/export.go b/ctl/export.go index 84b91edec..546d1ae1b 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -21,6 +21,8 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // ExportCommand represents a command for bulk exporting data from a server. @@ -38,7 +40,7 @@ type ExportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewExportCommand returns a new instance of ExportCommand. @@ -67,7 +69,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { if cmd.Path != "" { f, err := os.Create(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "creating file") } defer f.Close() @@ -77,7 +79,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Determine slice count. @@ -89,21 +91,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } if err != nil { - return err + return errors.Wrap(err, "getting slice count") } // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { - return err + return errors.Wrap(err, "exporting") } } // Close writer, if applicable. if w, ok := w.(io.Closer); ok { if err := w.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } } @@ -114,6 +116,6 @@ func (cmd *ExportCommand) TLSHost() string { return cmd.Host } -func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/export_test.go b/ctl/export_test.go index 401b902e9..48a90a065 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -59,14 +59,10 @@ func TestExportCommand_Run(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.URI = uri - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) @@ -75,8 +71,7 @@ func TestExportCommand_Run(t *testing.T) { cm.Index = "i" cm.Frame = "f" cm.View = pilosa.ViewStandard - err = cm.Run(context.Background()) - if err != nil { + if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } } diff --git a/ctl/generate_config.go b/ctl/generate_config.go index 5ddef99b1..a64678475 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -43,7 +43,6 @@ bind = "localhost:10101" max-writes-per-request = 5000 [cluster] - poll-interval = "2m0s" replicas = 1 hosts = [ "localhost:10101", diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 801ab4055..56f392bba 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -17,7 +17,6 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" @@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { + } else if !strings.Contains(buf.String(), "localhost:10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import.go b/ctl/import.go index 8103bf673..004e47e96 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -17,7 +17,6 @@ package ctl import ( "context" "encoding/csv" - "errors" "fmt" "io" "log" @@ -27,6 +26,8 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // ImportCommand represents a command for bulk importing data. @@ -66,7 +67,7 @@ type ImportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewImportCommand returns a new instance of ImportCommand. @@ -93,14 +94,14 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } cmd.Client = client if cmd.CreateSchema { err := cmd.ensureSchema(ctx) if err != nil { - return err + return errors.Wrap(err, "ensuring schema") } } @@ -152,7 +153,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -172,7 +173,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -242,7 +243,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err logger.Printf("importing slice: %d, n=%d", slice, len(bits)) if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil { - return err + return errors.Wrap(err, "importing") } } @@ -259,7 +260,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -279,7 +280,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -339,7 +340,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er logger.Printf("importing keys: n=%d", len(bits)) if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { - return err + return errors.Wrap(err, "importing keys") } return nil @@ -355,7 +356,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er // Open file for reading. f, err := os.Open(path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() @@ -375,7 +376,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er if err == io.EOF { break } else if err != nil { - return err + return errors.Wrap(err, "reading") } // Ignore blank rows. @@ -436,7 +437,7 @@ func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.F logger.Printf("importing slice: %d, n=%d", slice, len(vals)) if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil { - return err + return errors.Wrap(err, "importing values") } } @@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string { return cmd.Host } -func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/import_test.go b/ctl/import_test.go index 5979bdbee..08eea8323 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -65,14 +65,10 @@ func TestImportCommand_Run(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.URI = uri - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() cm.Index = "i" @@ -85,10 +81,8 @@ func TestImportCommand_Run(t *testing.T) { } } -// Ensure that the ImportValue path runs (note: we have specifed a value -// for cm.Field. Because the handler doesn't return errors (it sends them -// to the logger), we don't get an error returned at `cm.Run()` even though -// we haven't setup frame `f` to be RangeEnabled. +// Ensure that the ImportValue path runs (note: we have specified a value +// for cm.Field.) func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} @@ -105,18 +99,14 @@ func TestImportCommand_RunValue(t *testing.T) { defer hldr.Close() s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.URI = uri - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" @@ -133,7 +123,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = pilosa.DefaultHost + cm.Host = "anyhost" cm.Index = "i" cm.Frame = "f" file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/inspect.go b/ctl/inspect.go index dd59bf429..943ab6e57 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // InspectCommand represents a command for inspecting fragment data files. @@ -49,19 +50,19 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { // Open file handle. f, err := os.Open(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() fi, err := f.Stat() if err != nil { - return err + return errors.Wrap(err, "statting file") } // Memory map the file. data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { - return err + return errors.Wrap(err, "mmapping") } defer syscall.Munmap(data) @@ -70,7 +71,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error { fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...") bm := roaring.NewSliceBitmap() if err := bm.UnmarshalBinary(data); err != nil { - return err + return errors.Wrap(err, "unmarshalling") } fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) diff --git a/ctl/restore.go b/ctl/restore.go index 38528c2b5..ef863096e 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -16,11 +16,12 @@ package ctl import ( "context" - "errors" "io" "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // RestoreCommand represents a command for restoring a frame from a backup. @@ -39,7 +40,7 @@ type RestoreCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewRestoreCommand returns a new instance of RestoreCommand. @@ -59,19 +60,19 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error { // Create a client to the server. client, err := CommandClient(cmd) if err != nil { - return err + return errors.Wrap(err, "creating client") } // Open backup file. f, err := os.Open(cmd.Path) if err != nil { - return err + return errors.Wrap(err, "opening file") } defer f.Close() // Restore backup file to the cluster. if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return err + return errors.Wrap(err, "restoring") } return nil @@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string { return cmd.Host } -func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 9dd2d3661..896db57c5 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -48,14 +48,10 @@ func TestRestoreCommand_Run(t *testing.T) { s := test.NewServer() defer s.Close() - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - s.Handler.URI = uri - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Holder = hldr.Holder + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewRestoreCommand(stdin, stdout, stderr) cm.Path = file.Name() diff --git a/ctl/server.go b/ctl/server.go index 78f72b732..4816f6170 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -24,26 +24,41 @@ import ( // BuildServerFlags attaches a set of flags to the command for a server instance. func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() - flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") - flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") + flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") - flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") - flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") - flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") - flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") - flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") - flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") - flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.") - flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") + flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") + flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging") + + // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) + + // Cluster + flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") + flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.") + flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") + flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") + flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + + // Gossip + flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") + flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") + flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.") + flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.") + flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.") + + // AntiEntropy + flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.") + + // Metric + flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") + flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") + flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") } diff --git a/ctl/server_test.go b/ctl/server_test.go index dc31426ac..9866b497d 100644 --- a/ctl/server_test.go +++ b/ctl/server_test.go @@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) { stdin, stdout, stderr := GetIO(buf) Server := server.NewCommand(stdin, stdout, stderr) BuildServerFlags(cm, Server) - if cm.Flags().Lookup("gossip-port").Name == "" { - t.Fatal("gossip-port flag is required") - } if cm.Flags().Lookup("data-dir").Name == "" { t.Fatal("data-dir flag is required") } diff --git a/diagnostics.go b/diagnostics.go new file mode 100644 index 000000000..bfebbb495 --- /dev/null +++ b/diagnostics.go @@ -0,0 +1,309 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// Default version check URL. +const ( + defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" +) + +type versionResponse struct { + Version string `json:"version"` + Message string `json:"message"` +} + +// DiagnosticsCollector represents a collector/sender of diagnostics data. +type DiagnosticsCollector struct { + mu sync.Mutex + host string + VersionURL string + version string + lastVersion string + startTime int64 + start time.Time + + metrics map[string]interface{} + + client *http.Client + + Logger Logger + + server *Server +} + +// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". +func NewDiagnosticsCollector(host string) *DiagnosticsCollector { + return &DiagnosticsCollector{ + host: host, + VersionURL: defaultVersionCheckURL, + startTime: time.Now().Unix(), + start: time.Now(), + client: &http.Client{Timeout: 10 * time.Second}, + metrics: make(map[string]interface{}), + Logger: NopLogger, + } +} + +// SetVersion of locally running Pilosa Cluster to check against master. +func (d *DiagnosticsCollector) SetVersion(v string) { + d.version = v + d.Set("Version", v) +} + +// Flush sends the current metrics. +func (d *DiagnosticsCollector) Flush() error { + d.mu.Lock() + defer d.mu.Unlock() + d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) + buf, err := d.encode() + if err != nil { + return err + } + req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + resp, err := d.client.Do(req) + if err != nil { + return err + } + // Intentionally ignoring response body, as user does not need to be notified of error. + defer resp.Body.Close() + return nil +} + +// CheckVersion of the local build against Pilosa master. +func (d *DiagnosticsCollector) CheckVersion() error { + var rsp versionResponse + req, err := http.NewRequest("GET", d.VersionURL, nil) + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return fmt.Errorf("json decode: %s", err) + } + + // If version has not changed since the last check, return + if rsp.Version == d.lastVersion { + return nil + } + + d.lastVersion = rsp.Version + if err := d.compareVersion(rsp.Version); err != nil { + d.Logger.Printf("%s\n", err.Error()) + } + + return nil +} + +// compareVersion check version strings. +func (d *DiagnosticsCollector) compareVersion(value string) error { + currentVersion := versionSegments(value) + localVersion := versionSegments(d.version) + + if localVersion[0] < currentVersion[0] { //Major + return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) + } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor + return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) + } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch + return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) + } + + return nil +} + +// Encode metrics maps into the json message format. +func (d *DiagnosticsCollector) encode() ([]byte, error) { + return json.Marshal(d.metrics) +} + +// Set adds a key value metric. +func (d *DiagnosticsCollector) Set(name string, value interface{}) { + switch v := value.(type) { + case string: + if v == "" { + // Do not set empty string + return + } + } + d.mu.Lock() + defer d.mu.Unlock() + d.metrics[name] = value +} + +// logErr logs the error and returns true if an error exists +func (d *DiagnosticsCollector) logErr(err error) bool { + if err != nil { + d.Logger.Printf("%v", err) + return true + } + return false +} + +// EnrichWithOSInfo adds OS information to the diagnostics payload. +func (d *DiagnosticsCollector) EnrichWithOSInfo() { + uptime, err := d.server.systemInfo.Uptime() + if !d.logErr(err) { + d.Set("HostUptime", uptime) + } + platform, err := d.server.systemInfo.Platform() + if !d.logErr(err) { + d.Set("OSPlatform", platform) + } + family, err := d.server.systemInfo.Family() + if !d.logErr(err) { + d.Set("OSFamily", family) + } + version, err := d.server.systemInfo.OSVersion() + if !d.logErr(err) { + d.Set("OSVersion", version) + } + kernelVersion, err := d.server.systemInfo.KernelVersion() + if !d.logErr(err) { + d.Set("OSKernelVersion", kernelVersion) + } +} + +// EnrichWithMemoryInfo adds memory information to the diagnostics payload. +func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { + memFree, err := d.server.systemInfo.MemFree() + if !d.logErr(err) { + d.Set("MemFree", memFree) + } + memTotal, err := d.server.systemInfo.MemTotal() + if !d.logErr(err) { + d.Set("MemTotal", memTotal) + } + memUsed, err := d.server.systemInfo.MemUsed() + if !d.logErr(err) { + d.Set("MemUsed", memUsed) + } +} + +// EnrichWithSchemaProperties adds schema info to the diagnostics payload. +func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { + var numSlices uint64 + numFrames := 0 + numIndexes := 0 + bsiFieldCount := 0 + timeQuantumEnabled := false + + for _, index := range d.server.Holder.Indexes() { + numSlices += index.MaxSlice() + 1 + numIndexes += 1 + for _, frame := range index.Frames() { + numFrames += 1 + if fields, err := frame.GetFields(); err == nil { + bsiFieldCount += len(fields) + } + if frame.TimeQuantum() != "" { + timeQuantumEnabled = true + } + } + } + + d.Set("NumIndexes", numIndexes) + d.Set("NumFrames", numFrames) + d.Set("NumSlices", numSlices) + d.Set("BSIFieldCount", bsiFieldCount) + d.Set("TimeQuantumEnabled", timeQuantumEnabled) +} + +// versionSegments returns the numeric segments of the version as a slice of ints. +func versionSegments(segments string) []int { + segments = strings.Trim(segments, "v") + segments = strings.Split(segments, "-")[0] + s := strings.Split(segments, ".") + segmentSlice := make([]int, len(s)) + for i, v := range s { + segmentSlice[i], _ = strconv.Atoi(v) + } + return segmentSlice +} + +// SystemInfo collects information about the host OS. +type SystemInfo interface { + Uptime() (uint64, error) + Platform() (string, error) + Family() (string, error) + OSVersion() (string, error) + KernelVersion() (string, error) + MemFree() (uint64, error) + MemTotal() (uint64, error) + MemUsed() (uint64, error) +} + +// NewNopSystemInfo creates a no-op implementation of SystemInfo. +func NewNopSystemInfo() *NopSystemInfo { + return &NopSystemInfo{} +} + +// NopSystemInfo is a no-op implementation of SystemInfo. +type NopSystemInfo struct { +} + +// Uptime is a no-op implementation of SystemInfo.Uptime. +func (n *NopSystemInfo) Uptime() (uint64, error) { + return 0, nil +} + +// Platform is a no-op implementation of SystemInfo.Platform. +func (n *NopSystemInfo) Platform() (string, error) { + return "", nil +} + +// Family is a no-op implementation of SystemInfo.Family. +func (n *NopSystemInfo) Family() (string, error) { + return "", nil +} + +// OSVersion is a no-op implementation of SystemInfo.OSVersion. +func (n *NopSystemInfo) OSVersion() (string, error) { + return "", nil +} + +// KernelVersion is a no-op implementation of SystemInfo.KernelVersion. +func (n *NopSystemInfo) KernelVersion() (string, error) { + return "", nil +} + +// MemFree is a no-op implementation of SystemInfo.MemFree. +func (n *NopSystemInfo) MemFree() (uint64, error) { + return 0, nil +} + +// MemTotal is a no-op implementation of SystemInfo.MemTotal. +func (n *NopSystemInfo) MemTotal() (uint64, error) { + return 0, nil +} + +// MemUsed is a no-op implementation of SystemInfo.MemUsed. +func (n *NopSystemInfo) MemUsed() (uint64, error) { + return 0, nil +} diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go deleted file mode 100644 index 60248d6bb..000000000 --- a/diagnostics/diagnostics.go +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package diagnostics - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "strconv" - "strings" - "sync" - "time" - - "github.com/shirou/gopsutil/host" - "github.com/shirou/gopsutil/mem" - "github.com/sony/gobreaker" -) - -// TODO: unique Cluster ID - -// Default version check URL. -const ( - DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version" -) - -type versionResponse struct { - Version string `json:"version"` - Message string `json:"message"` -} - -// Diagnostics represents a client to the Pilosa cluster. -type Diagnostics struct { - mu sync.Mutex - wg sync.WaitGroup - closing chan struct{} - host string - VersionURL string - version string - lastVersion string - startTime int64 - start time.Time - - metrics map[string]interface{} - - client *http.Client - interval time.Duration - - cb *gobreaker.CircuitBreaker - logOutput io.Writer -} - -// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port". -func New(host string) *Diagnostics { - - return &Diagnostics{ - closing: make(chan struct{}), - host: host, - VersionURL: DefaultVersionCheckURL, - startTime: time.Now().Unix(), - start: time.Now(), - client: http.DefaultClient, - metrics: make(map[string]interface{}), - logOutput: ioutil.Discard, - } -} - -// SetVersion of locally running Pilosa Cluster to check against master. -func (d *Diagnostics) SetVersion(v string) { - d.version = v - d.Set("Version", v) -} - -// SetInterval of the diagnostic go routine and match with the circuit breaker timeout. -func (d *Diagnostics) SetInterval(i time.Duration) { - d.interval = i -} - -// schedule start the diagnostics service ticker. -func (d *Diagnostics) schedule() { - ticker := time.NewTicker(d.interval) - defer ticker.Stop() - - for { - select { - case <-d.closing: - return - case <-ticker.C: - d.CheckVersion() - d.Flush() - } - } -} - -// Flush sends the current metrics. -func (d *Diagnostics) Flush() error { - d.mu.Lock() - d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) - buf, _ := d.Encode() - d.mu.Unlock() - - _, err := d.cb.Execute(func() (interface{}, error) { - req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - resp, err := d.client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // TODO verify response - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return body, nil - }) - - return err -} - -// Open configures the circuit breaker used by the HTTP client. -func (d *Diagnostics) Open() { - var st gobreaker.Settings - if d.interval > 0 { - st.Timeout = d.interval * 2 - } - d.cb = gobreaker.NewCircuitBreaker(st) - - d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics") -} - -// Close notify goroutine to stop. -func (d *Diagnostics) Close() error { - close(d.closing) - d.wg.Wait() - return nil -} - -// CheckVersion of the local build against Pilosa master. -func (d *Diagnostics) CheckVersion() error { - var rsp versionResponse - req, err := http.NewRequest("GET", d.VersionURL, nil) - resp, err := d.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return fmt.Errorf("json decode: %s", err) - } - - // Same a version as last test - if rsp.Version == d.lastVersion { - return nil - } - - d.lastVersion = rsp.Version - if err := d.CompareVersion(rsp.Version); err != nil { - d.logger().Printf("%s\n", err.Error()) - } - - return nil -} - -// CompareVersion check version strings. -func (d *Diagnostics) CompareVersion(value string) error { - currentVersion := VersionSegments(value) - localVersion := VersionSegments(d.version) - - if localVersion[0] < currentVersion[0] { //Major - return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor - return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) - } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch - return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) - } - - return nil -} - -// Encode metrics maps into the json message format. -func (d *Diagnostics) Encode() ([]byte, error) { - return json.Marshal(d.metrics) -} - -// Set adds a key value metric. -func (d *Diagnostics) Set(name string, value interface{}) { - d.mu.Lock() - defer d.mu.Unlock() - d.metrics[name] = value -} - -// SetLogger Set the logger output type. -func (d *Diagnostics) SetLogger(logger io.Writer) { - d.logOutput = logger -} - -// logger returns a logger that writes to LogOutput. -func (d *Diagnostics) logger() *log.Logger { - return log.New(d.logOutput, "", log.LstdFlags) -} - -// EnrichWithOSInfo adds OS information to the diagnostics payload. -func (d *Diagnostics) EnrichWithOSInfo() { - osInfo, err := host.Info() - if err != nil { - d.logOutput.Write([]byte(err.Error())) - } - d.Set("HostUptime", osInfo.Uptime) - - platform, family, version, err := host.PlatformInformation() - if err != nil { - d.logOutput.Write([]byte(err.Error())) - } - d.Set("OSPlatform", platform) - d.Set("OSFamily", family) - d.Set("OSVersion", version) - - kernelVersion, err := host.KernelVersion() - if err != nil { - d.logOutput.Write([]byte(err.Error())) - } - d.Set("OSKernelVersion", kernelVersion) -} - -// EnrichWithMemoryInfo adds memory information to the diagnostics payload. -func (d *Diagnostics) EnrichWithMemoryInfo() { - memory, err := mem.VirtualMemory() - if err != nil { - d.logOutput.Write([]byte(err.Error())) - } - d.Set("MemFree", memory.Free) - d.Set("MemTotal", memory.Total) - d.Set("MemUsed", memory.Used) - -} - -// VersionSegments returns the numeric segments of the version as a slice of ints. -func VersionSegments(segments string) []int { - segments = strings.Trim(segments, "v") - segments = strings.Split(segments, "-")[0] - s := strings.Split(segments, ".") - segmentSlice := make([]int, len(s)) - for i, v := range s { - segmentSlice[i], _ = strconv.Atoi(v) - } - return segmentSlice -} diff --git a/diagnostics/diagnostics_test.go b/diagnostics_internal_test.go similarity index 80% rename from diagnostics/diagnostics_test.go rename to diagnostics_internal_test.go index 8f85a57db..517dbed3d 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics_internal_test.go @@ -12,36 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -package diagnostics_test +package pilosa import ( "encoding/json" - "io/ioutil" "net/http" "net/http/httptest" "reflect" "runtime" "strings" "testing" - - "github.com/pilosa/pilosa/diagnostics" ) func TestDiagnosticsClient(t *testing.T) { // Mock server. server := httptest.NewServer(nil) - defer server.Close() // Create a new client. - d := diagnostics.New(server.URL) - d.SetLogger(ioutil.Discard) - d.Open() - defer d.Close() + d := NewDiagnosticsCollector(server.URL) d.Set("gg", 10) d.Set("ss", "ss") - data, err := d.Encode() + data, err := d.encode() if err != nil { t.Fatal(err) } @@ -58,7 +51,7 @@ func TestDiagnosticsClient(t *testing.T) { // Test the metrics after a flush. d.Flush() - data, err = d.Encode() + data, err = d.encode() if err != nil { t.Fatal(err) } @@ -74,7 +67,7 @@ func TestDiagnosticsClient(t *testing.T) { func TestDiagnosticsVersion_Parse(t *testing.T) { version := "0.1.1" - vs := diagnostics.VersionSegments(version) + vs := versionSegments(version) output := []int{0, 1, 1} if !reflect.DeepEqual(vs, output) { @@ -83,35 +76,33 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { } func TestDiagnosticsVersion_Compare(t *testing.T) { - d := diagnostics.New("localhost:10101") - d.Open() - defer d.Close() + d := NewDiagnosticsCollector("localhost:10101") version := "v0.1.1" d.SetVersion(version) - err := d.CompareVersion("v1.7.0") + err := d.compareVersion("v1.7.0") if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } - err = d.CompareVersion("1.7.0") + err = d.compareVersion("1.7.0") if !strings.Contains(err.Error(), "A newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } - err = d.CompareVersion("0.7.0") + err = d.compareVersion("0.7.0") if !strings.Contains(err.Error(), "The latest Minor release is") { t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) } - err = d.CompareVersion("0.1.2") + err = d.compareVersion("0.1.2") if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") { t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) } - err = d.CompareVersion("0.1.1") + err = d.compareVersion("0.1.1") if err != nil { t.Fatalf("Versions should match") } d.SetVersion("v1.7.0") - err = d.CompareVersion("0.7.2") + err = d.compareVersion("0.7.2") if err != nil { t.Fatalf("Local version is greater") } @@ -125,11 +116,9 @@ func TestDiagnosticsVersion_Check(t *testing.T) { Version: "1.1.1", }) })) - defer server.Close() // Create a new client. - d := diagnostics.New("localhost:10101") - defer d.Close() + d := NewDiagnosticsCollector("localhost:10101") version := "0.1.1" d.SetVersion(version) @@ -138,10 +127,6 @@ func TestDiagnosticsVersion_Check(t *testing.T) { d.CheckVersion() } -type versionResponse struct { - Version string `json:"version"` -} - func compareJSON(a, b []byte) (bool, error) { var j1, j2 interface{} if err := json.Unmarshal(a, &j1); err != nil { @@ -156,12 +141,9 @@ func compareJSON(a, b []byte) (bool, error) { func BenchmarkDiagnostics(b *testing.B) { // Mock server. server := httptest.NewServer(nil) - defer server.Close() // Create a new client. - d := diagnostics.New(server.URL) - d.SetLogger(ioutil.Discard) - defer d.Close() + d := NewDiagnosticsCollector(server.URL) prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..6a9f50b45 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,5 @@ +Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/). + +Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search. + +Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request! diff --git a/docs/administration.md b/docs/administration.md index 795d6f1c4..0ecbbed94 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -5,6 +5,7 @@ nav = [ "Installing in production", "Importing and Exporting Data", "Versioning", + "Resizing the Cluster", "Backup/restore", ] +++ @@ -47,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog #### Importing -The import API expects a csv of RowID,ColumnID's. +The import API expects a csv of rowID,columnID's. -When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. +When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. ``` pilosa import --sort -i project -f stargazer project-stargazer.csv @@ -64,12 +65,12 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count ```
-

Note that you must first create a frame with Range Encoding enabled and a field. View Create Frame for more details.

+

Note that you must first create a frame with range-encoding enabled and a field. View Create Frame for more details.

#### Exporting -Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. +Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID. ``` curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \ --header "Accept: text/csv" @@ -91,7 +92,106 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ #### Upgrading -When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. +To upgrade Pilosa: + +1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`. +2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases). +3. Shut down the Pilosa cluster. +4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node. +5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make. +6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes. + +##### Version 0.9 + +Pilosa v0.9 introduces a few compatibility changes that need to be addressed. + +**Configuration changes**: These changes need to occur before starting Pilosa v0.9: + +1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster. +2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set. +3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds. + +**Data directory changes**: These changes need to occur while the cluster is shut down, before starting Pilosa v0.9: + +Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`: + +1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. +2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`. +3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`. +4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position. +5. Copy the file `topology` into the data directories of every node in the cluster, naming it `.topology` (note the dot), e.g. `cp topology ~/.pilosa/.topology` or `scp topology node1:.pilosa/.topology`. +6. Copy the node ID files into the respective node data directories. For example, `node0.id` will be placed on the first node in the `cluster.hosts` list, with the name `.id`. For example: `scp node0.id node0:.pilosa/.id`. Again, it is very important that the ordering you give the nodes with these IDs matches the ordering you had in your existing `cluster.hosts` setting. + +**Application changes**: + +1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported. +2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed. +3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release. + +### Resizing the Cluster + +If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process. + +#### Adding a Node + +You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries. + +If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately. + +#### Removing a Node + +In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself. + +To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing an `/id` request to the node: +``` request +curl localhost:10102/id +``` +``` response +40a891fa-243b-4d71-ae24-4f5c78a0f4b1 +``` + +If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node: +``` request +curl localhost:10101/status +``` +``` response +{ + "state":"NORMAL", + "nodes":[ + {"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}} + {"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}} + {"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}} + ] +} +``` + +Once you have the ID of the node that you want to remove from the cluster, issue the following request: +``` +curl localhost:10101/cluster/resize/remove-node \ + -X POST \ + -d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}' +``` +At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries. + +Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator). + +#### Aborting a Resize Job + +If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node. +For example, if your coordinator node is `localhost:10101`, then you can run: +``` +curl localhost:10101/cluster/resize/abort -X POST +``` +This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began. + +#### Changing the Coordinator + +In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator. +``` +curl localhost:10101/cluster/resize/set-coordinator \ + -X POST \ + -d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}' +``` ### Backup/restore @@ -112,19 +212,19 @@ Note: This will only work when the replication factor is >= 2 #### Copying data files manually -- To accomplish this goal you will 1st need: - - List of all Indexes on your cluster - - List of all frames in your Indexes - - Max slice per Index, listed in the /status endpoint +- To accomplish this you will first need: + - List of all indexes on your cluster + - List of all frames in your indexes + - Max slice per index, listed in the /status endpoint - With this information you can query the `/fragment/nodes` endpoint and iterate over each slice - Using the list of slices owned by this node you will then need to manually: - setup a directory structure similar to the other nodes with a path for each Index/Frame - copy each owned slice for an existing node to this new node - Modify the cluster config file to replace the previous node address with the new node address. - Restart the cluster -- Wait for the 1st sync (10 minutes) to validate Index connections +- Wait for the first sync (10 minutes) to validate Index connections -#### Diagnostics +### Diagnostics Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions. @@ -135,7 +235,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumCPU:** Number of Cores per Node - **BSIEnabled:** Bit Slice Index Frames in use. - **TimeQuantumEnabled:** Time Quantum Frames in use. -- **InverseEnabled:** Inverse Frames in use. - **NumIndexes:** Number of Indexes in the Cluster. - **NumFrames:** Number of Frames in the Cluster. - **NumSlices:** Number of Slices in the Cluster. @@ -145,16 +244,16 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. -#### Metrics +### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - Host to receive events - - Polling interval for runtime metrics - - Metric type (StatsD, Expvar). + - [Host](../configuration/#metric-host): specify host that receives metric events + - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration/#metric-service): declare type StatsD or Expvar -##### Tags +#### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - NodeID @@ -163,7 +262,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - View - Slice -##### Events +#### Events We currently track the following events - **Index:** The creation of a new Index. diff --git a/docs/api-reference.md b/docs/api-reference.md index 206c30f55..2e744954e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -13,13 +13,10 @@ nav = [] Returns the schema of all indexes in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index ``` - -Response: -``` +``` response {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` @@ -29,13 +26,10 @@ Response: Returns the schema of the specified index in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user ``` - -Response: -``` +``` response {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` @@ -45,13 +39,10 @@ Response: Creates an index with the given name. -Request: -``` +``` request curl -XPOST localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -61,13 +52,10 @@ Response: Removes the given index. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -77,15 +65,12 @@ Response: Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. -Request: -``` +``` request curl localhost:10101/index/user/query \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' -``` - -Response: + -d 'Bitmap(frame="language", row=5)' ``` +``` response {"results":[{"attrs":{},"bits":[100]}]} ``` @@ -93,16 +78,14 @@ In order to send protobuf binaries in the request and response, set `Content-Typ The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`. -The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -Request: -``` +``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' -``` -Response: + -d 'Bitmap(frame="language", row=5)' ``` +``` response { "results":[{"attrs":{},"bits":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] @@ -119,14 +102,14 @@ Creates a frame in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: -* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. -* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. +* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -* `rangeEnabled` (boolean): Enables range-encoded fields in this frame. -* `fields` (array): List of range-encoded fields. +* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. +* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: + * `name` (string): Field name. * `type` (string): Field type, currently only "int" is supported. * `min` (int): Minimum value allowed for this field. @@ -134,21 +117,19 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. -Request: +``` request +curl localhost:10101/index/user/frame/language -X POST ``` -curl localhost:10101/index/user/frame/language \ - -X POST \ - -d '{"options": {"inverseEnabled": true}}' +``` response +{} ``` -``` +``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ - -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' -``` - -Response: + -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` +``` response {} ``` @@ -158,45 +139,10 @@ Response: Removes the given frame. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/frame/language ``` - -Response: -``` -{} -``` - -### Change frame time quantum - -`PATCH /index//frame//time-quantum` - -Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. - -The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: - -* (Empty string) -* Y: year -* M: month -* D: day -* H: hour -* YM: year and month -* MD: month and day -* DH: day and hour -* YMD: year, month and day -* MDH: month, day and hour -* YMDH: year, month, day and hour - -Request: -``` -curl localhost:10101/index/user/frame/language/time-quantum \ - -X POST \ - -d '{"timeQuantum": "YM"}' -``` - -Response: -``` +``` response {} ``` @@ -211,134 +157,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max` * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. -Request: -``` +``` request curl localhost:10101/index/repository/frame/stats/field/pullrequests \ -X POST \ -d '{"type": "int", "min": 0, "max": 1000000}' ``` - -Response: -``` -{} -``` - -### Create input definition - -`POST /index//input-definition/` - -Creates an input definition in the given index with the given name. - -The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: - -* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. -* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. -* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. - -The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. - -* `name` (string): Maps the source data field to actions that process the field's corresponding value. -* `actions` (array): List of actions that will process the field's value. - -The `action` describes how the field value will be processed. Each `action` may contain: - -* `frame` (string): The Frame that will contain this action's set bits. -* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame. -* `valueDestination` (string): The mapping rule used for this data. - - `value-to-row`: The value should be an integer and will map directly to a RowID. - - `single-row-boolean`: If the value is true set a bit using the `rowid`. - - `mapping`: Map the value to a RowID in the `valueMap`. -* `valueMap` (object): string and integer pairs used to map field values to RowID's. - -Request: -``` -curl localhost:10101/index/user/input-definition/stargazer-input \ - -X POST \ - -d '{ - "frames":[ - { - "name": "language", - "options": {"inverseEnabled": true} - } - ], - "fields":[ - { - "name": "repo_id", - "primaryKey":true - }, - { - "name": "language_id", - "actions":[ - { - "frame": "language", - "valueDestination": "mapping", - "valueMap": { - "Go": 5, - "Python": 17, - "C++": 10 - } - } - ] - } - ] - }' -``` - -Response: -``` -{} -``` - -### Get input definition - -`GET /index//input-definition/` - -Returns the given input definition as JSON. - -Request: -``` -curl -XGET localhost:10101/index/user/input-definition/stargazer-input -``` - -Response: -``` -{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} -``` - -### Remove input definition - -`DELETE /index//input-definition/` - -Removes the given input definition. - -Request: -``` -curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input -``` - -Response: -``` -{} -``` - -### Process input data - -`POST /index//input/` - -Processes the JSON payload using the given input definition. - -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. - -Request: -``` -curl localhost:10101/index/user/input/stargazer-input \ - -X POST \ - -d '[{"language_id": "Go", "repo_id": 92274475}]' -``` - -Response: -``` +``` response {} ``` @@ -348,13 +172,10 @@ Response: Returns the hosts in the cluster. -Request: -``` +``` request curl -XGET localhost:10101/hosts ``` - -Response: -``` +``` response [{"host":":10101"}] ``` @@ -364,13 +185,10 @@ Response: Returns the version of the Pilosa server. -Request: -``` +``` request curl -XGET localhost:10101/version ``` - -Response: -``` +``` response {"version":"v0.6.0"} ``` diff --git a/docs/architecture.md b/docs/architecture.md index 5cc5ff6ee..81a7e54f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,5 +18,8 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B * After the container storage section is an operation log, of unspecified length. ![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png) +*Pilosa Roaring storage format diagram* All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header. + +Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa. diff --git a/docs/client-libraries.md b/docs/client-libraries.md index 67e36b5dc..d5074a059 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -10,12 +10,13 @@ nav = [ ## Client Libraries +This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is not wise to use the output of a hash, or randomly distributed ids with Pilosa. ### Go You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. @@ -95,9 +96,9 @@ func main() { ### Python -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions. +You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. @@ -170,7 +171,7 @@ client.query(stargazer.setbit(99999, 77777)) You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. -We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. +We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. Error handling has been omitted in the example below for brevity. diff --git a/docs/configuration.md b/docs/configuration.md index 43b70c8f1..2d9d109b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,24 +27,18 @@ Every command line flag has a corresponding environment variable. The environmen ### Config file -The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file: +The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file: ```toml [cluster] - poll-interval = "2m0s" + coordinator = true replicas = 1 ``` -Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes: -```toml -[cluster] - hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"] -``` - ### All Options #### Anti Entropy Interval -* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync. +* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync. * Flag: `--anti-entropy.interval="10m0s"` * Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"` * Config: @@ -84,7 +78,18 @@ Any flag that has a value that is a comma separated list on the command line bec * Config: ```toml - log_path = "/path/to/logfile" + log-path = "/path/to/logfile" + ``` + +#### Verbose + +* Description: Enable verbose logging. +* Flag: `--verbose` +* Env: `PILOSA_VERBOSE` +* Config: + + ```toml + verbose = true ``` #### Max Writes Per Request @@ -110,16 +115,16 @@ Any flag that has a value that is a comma separated list on the command line bec port = 11101 ``` -#### Gossip Seed +#### Gossip Seeds -* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. -* Flag: `--gossip.seed="localhost:11101"` -* Env: `PILOSA_GOSSIP_SEED="localhost:11101"` +* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms. +* Flag: `--gossip.seeds="localhost:11101"` +* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101"` * Config: ```toml [gossip] - seed = "localhost:11101" + seeds = ["localhost:11101"] ``` #### Gossip Key @@ -133,28 +138,16 @@ Any flag that has a value that is a comma separated list on the command line bec key = "/var/secret/gossip.key32" ``` -#### Cluster Hosts +#### Cluster Coordinator -* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. -* Flag: `--cluster.hosts="localhost:10101"` -* Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"` +* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator. +* Flag: `cluster.coordinator` +* Env: `PILOSA_CLUSTER_COORDINATOR` * Config: ```toml [cluster] - hosts = ["localhost:10101"] - ``` - -#### Cluster Poll Interval - -* Description: Polling interval for cluster. -* Flag: `cluster.poll-interval="1m0s"` -* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"` -* Config: - - ```toml - [cluster] - poll-interval = "1m0s" + coordinator = true ``` #### Cluster Long Query Time @@ -183,9 +176,8 @@ Any flag that has a value that is a comma separated list on the command line bec #### Cluster Type -* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]. +* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip]. * static - Messaging between nodes is disabled. This is primarily used for testing. - * http - Messages are transmitted over HTTP. * gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip. * Flag: `cluster.type="gossip"` * Env: `PILOSA_CLUSTER_TYPE="gossip"` @@ -205,12 +197,12 @@ Any flag that has a value that is a comma separated list on the command line bec ```toml [profile] - cpu = "/path/to/somewhere" + cpu = "/path/to/somewhere" ``` #### Profile CPU Time -* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set. +* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set. * Flag: `--profile.cpu-time="30s"` * Env: `PILOSA_PROFILE_CPU_TIME="30s" * Config: @@ -219,8 +211,9 @@ Any flag that has a value that is a comma separated list on the command line bec [profile] cpu-time = "30s" ``` -##### Metric Service -* Description: Which stats service to use (StatsD or ExpVar). + +#### Metric Service +* Description: Which stats service to use. Choose from [statsd, expvar, none]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: @@ -230,7 +223,7 @@ Any flag that has a value that is a comma separated list on the command line bec service = “statsd” ``` -##### Metric Host +#### Metric Host * Description: Address of the StatsD service host. * Flag: `--metric.host=localhost:8125` * Env: `PILOSA_METRIC_HOST=localhost:8125' @@ -241,9 +234,9 @@ Any flag that has a value that is a comma separated list on the command line bec host = "localhost:8125" ``` -##### Metric Poll Interval +#### Metric Poll Interval -* Description: Polling interval for runtime metrics. +* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected. * Flag: `metric.poll-interval=”0m15s”` * Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s` * Config: @@ -253,9 +246,9 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "0m15s" ``` -##### Metric Diagnostics +#### Metric Diagnostics -* Description: Enable diagnostic reporting. To disable diagnostics set to false. +* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false. * Flag: `metric.diagnostics` * Env: `PILOSA_METRIC_DIAGNOSTICS` * Config: @@ -266,7 +259,7 @@ Any flag that has a value that is a comma separated list on the command line bec ``` -##### TLS Certificate +#### TLS Certificate * Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions. * Flag: `tls.certificate=/srv/pilosa/certs/server.crt` @@ -278,7 +271,7 @@ Any flag that has a value that is a comma separated list on the command line bec certificate = "/srv/pilosa/certs/server.crt" ``` -##### TLS Certificate Key +#### TLS Certificate Key * Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension. * Flag: `tls.key=/srv/pilosa/certs/server.key` @@ -290,7 +283,7 @@ Any flag that has a value that is a comma separated list on the command line bec key = "/srv/pilosa/certs/server.key" ``` -##### TLS Skip Verify +#### TLS Skip Verify * Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial. * Flag: `tls.skip-verify` @@ -317,8 +310,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = true #### Node 1 @@ -331,8 +323,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = false #### Node 2 @@ -345,8 +336,7 @@ A three node cluster running on different hosts could be minimally configured as [cluster] replicas = 1 - type = "gossip" - hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + coordinator = false ### Example Cluster Configuration (HTTPS) @@ -365,8 +355,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = true [tls] certificate = "/home/pilosa/private/server.crt" @@ -384,8 +373,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -403,8 +391,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [cluster] replicas = 1 - type = "gossip" - hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"] + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -426,8 +413,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = true [tls] certificate = "/home/pilosa/private/server.crt" @@ -445,8 +431,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" @@ -464,8 +449,7 @@ You can run a cluster on the same host using the configuration above with a few [cluster] replicas = 1 - type = "gossip" - hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"] + coordinator = false [tls] certificate = "/home/pilosa/private/server.crt" diff --git a/docs/data-model.md b/docs/data-model.md index 5b1741fca..8375ae15f 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -24,7 +24,10 @@ Rows and columns can represent anything (they could even represent the same set Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row. -![data model diagram](/img/docs/data-model.svg) +Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen. + +![basic data model diagram](/img/docs/data-model.svg) +*Basic data model diagram* ### Index @@ -49,16 +52,18 @@ Row attributes are namespaced at the Frame level. Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. ![ranked frame diagram](/img/docs/frame-ranked.svg) +*Ranked frame diagram* #### LRU The LRU cache maintains the most recently accessed Rows. ![lru frame diagram](/img/docs/frame-lru.svg) +*LRU frame diagram* ### Time Quantum -Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. ### Attribute @@ -72,42 +77,30 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice. ### View -Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. +Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. #### Standard The standard View contains the same Row/Column format as the input data. -#### Inverse - -The Inverse View contains the same data with the Row and Column swapped. - -For example, the following `SetBit()` queries will result in the data described in the illustration below: -``` -SetBit(frame="A", rowID=8, columnID=3) -SetBit(frame="A", rowID=11, columnID=3) -SetBit(frame="A", rowID=19, columnID=5) -``` - -![inverse frame diagram](/img/docs/frame-inverse.svg) - #### Time Quantums If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: ``` -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00") -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") ``` ![time quantum frame diagram](/img/docs/frame-time-quantum.svg) +*Time quantum frame diagram* #### BSI Range-Encoding Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded -bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate, aggregate, and range queries on these BSI integers. +bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: @@ -120,4 +113,7 @@ SetFieldValue(col=2, frame="A", field1=1) SetFieldValue(col=3, frame="A", field1=6) ``` -![BSI diagram](/img/docs/frame-bsi.svg) +![BSI frame diagram](/img/docs/frame-bsi.svg) +*BSI frame diagram* + +Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. diff --git a/docs/examples.md b/docs/examples.md index bf5e76f29..d7830a2a8 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -17,7 +17,7 @@ New York City released an extremely detailed data set of over 1 billion taxi rid Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). -We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. +We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. @@ -163,7 +163,7 @@ durm := pdk.CustomMapper{ #### Import process -After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. +After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk/) for more details on this process. #### Queries @@ -201,11 +201,15 @@ For more examples and details, see this [ipython notebook](https://github.com/pi ### Chemical similarity search +
+This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API. +
+ #### Overview The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. -We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. ``` @@ -214,11 +218,11 @@ T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. -All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase +All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase). #### Data model -We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. +We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. @@ -263,16 +267,15 @@ python import_from_sdf.py -p -file id_fingerprint.csv ``` -First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 ``` curl localhost:10101/index/mole \ - -X POST \ - -d '{"options": {"columnLabel": "position_id"}}' + -X POST curl localhost:10101/index/mole/frame/fingerprint \ -X POST \ - -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + -d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' ``` @@ -302,7 +305,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a * Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id ```python - bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] bitmap_string = ', '.join(bit_maps) intersection = "Intersect(%s)" % bitmap_string mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] @@ -312,7 +315,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a ```python for m in mole_ids: - mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] existed_mol = False if len(mol) == len(fp): found = m @@ -331,7 +334,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN * Query Pilosa’s TopN to get list of similarity chembl_id ```python - query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 35042d59d..bee165643 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -4,7 +4,6 @@ weight = 3 nav = [ "Starting Pilosa", "Sample Project", - "Input Definition", "What's Next?", ] +++ @@ -20,7 +19,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th ### Starting Pilosa -Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa. +Follow the steps in the [Install](../installation/) document to install Pilosa. Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): ``` pilosa server @@ -31,49 +30,63 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest ``` Let's make sure Pilosa is running: -``` +``` request curl localhost:10101/status ``` - -Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` +``` response +{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]} +``` ### Sample Project In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. -Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation. +Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation. #### Create the Schema Note: -The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: -``` +The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: + +``` request curl localhost:10101/schema +``` +``` response {"indexes":null} ``` Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: -``` +``` request curl localhost:10101/index/repository -X POST ``` +``` response +{} +``` Let's create the `stargazer` frame which has user IDs of stargazers as its rows: -``` +``` request curl localhost:10101/index/repository/frame/stargazer \ -X POST \ -d '{"options": {"timeQuantum": "YMD"}}' ``` +``` response +{} +``` Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` frame, which will contain IDs for programming languages: +``` request +curl localhost:10101/index/repository/frame/language \ + -X POST ``` -curl localhost:10101/index/repository/frame/language -X POST +``` response +{} ``` #### Import Data From CSV Files -If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo. +Download the `stargazer.csv` and `language.csv` files here: ``` curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv @@ -95,10 +108,7 @@ docker cp language.csv pilosa:/language.csv docker exec -it pilosa /pilosa import -i repository -f language /language.csv ``` -Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages. - -### Input Definition -Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. +Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages. #### Make Some Queries @@ -107,46 +117,117 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d Which repositories did user 14 star: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(frame="stargazer", rowID=14)' + -d 'Bitmap(frame="stargazer", row=14)' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + } + ] +} ``` What are the top 5 languages in the sample data: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'TopN(frame="language", n=5)' ``` +``` response +{ + "results":[ + [ + {"id":5,"count":119}, + {"id":1,"count":50}, + {"id":4,"count":48}, + {"id":9,"count":31}, + {"id":13,"count":25} + ] + ] +} +``` Which repositories were starred by user 14 and 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Intersect( + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,3,362,396,416,461,464,466,470,486] + } + ] +} ``` Which repositories were starred by user 14 or 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Union( + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + } + ] +} ``` Which repositories were starred by user 14 and 19 and also were written in language 1: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))' + -d 'Intersect( + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19), + Bitmap(frame="language", row=1) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,362,416,461] + } + ] +} ``` Set user 99999 as a stargazer for repository 77777: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' + -d 'SetBit(frame="stargazer", column=77777, row=99999)' ``` +``` response +{"results":[true]} +``` + +Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number. +Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to poor performance, out of memory errors, and more. + + ### What's Next? diff --git a/docs/glossary.md b/docs/glossary.md index 50906a0cf..9b883017d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,53 +6,66 @@ nav = [] ## Glossary -Index: Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes. +[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. -Column: Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index. +[Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. -Row: Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index. +[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). -Bit: A bit is the intersection of a Row and Column. +[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. -Bitmap: The on-disk and in-memory representation of a Row. +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. -Roaring Bitmap: [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses. +Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. -Attribute: Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store. +[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). -PQL: Pilosa Query Language +[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries. -Index: The Index represents a data namespace. +Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). -Frame: Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame. +[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. -View: Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. -Fragment: A Fragment is the intersection of a frame and slice in an index. +[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -Slice: Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash. +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). -SliceWidth: This is the default number of columns in a slice. +MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. -MaxSlice: The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries. +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). -Anti-entropy: A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies. +Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). -Node: An individual running instance of Pilosa server which belongs to a cluster. +Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data. -Cluster: A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. +[PQL](../query-language/): Pilosa Query Language. -TopN: Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row. +[Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -Tanimoto: Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity. +[Range](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). -Protobuf:: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. +[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). -TOML: We use [TOML](https://github.com/toml-lang/toml) for our configuration file format. +[Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. -Jump Consistent Hash: A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -https://arxiv.org/pdf/1406.2294v1.pdf +[Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. -Partition: The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography. +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap). -Replica: A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made). +[Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). + +SliceWidth: This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. + +[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field). + +[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). + +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for time [Range](#range) queries. + +[TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). + +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). + +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/input-definition.md b/docs/input-definition.md deleted file mode 100644 index 7d50c6f9d..000000000 --- a/docs/input-definition.md +++ /dev/null @@ -1,64 +0,0 @@ -+++ -title = "Input Definition" -weight = 8 -nav = [ - "Create the Schema", - "Import Data", -] -+++ - -## Input Definition -This document builds on the data import concepts introduced in [Getting Started](../getting-started/). -Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data. - -### Create the Schema - -Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations. - -Before creating a schema, let's create the repository index first: - -``` -curl localhost:10101/index/repository -X POST -``` -The sample input definition schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started) in the `input_definition.json` file. Download it using: -``` -curl -OL https://github.com/pilosa/getting-started/raw/master/input_definition.json -``` - -Run the following to create the input definition: -``` -curl localhost:10101/index/repository/input-definition/stargazer -d @input_definition.json -``` - -Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition. -We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination: - - - value-to-row: The value for this field is used as the `rowID`. - - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame - -### Import Data - -The sample data for the input definition we created above is in the `json_input.json` file at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download it using: -``` -curl -OL https://github.com/pilosa/getting-started/raw/master/json_input.json -``` - -Then run the following to import it: -``` -curl localhost:10101/index/repository/input/stargazer -d @json_input.json -``` - -As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. -The data input above is equivalent to the following `SetBit()` operations: - -``` -curl localhost:10101/index/repository/query \ - -X POST \ - -d 'SetBit(frame="stargazer", columnID=91720568, rowID=513114) - SetBit(frame="stargazer", columnID=91720568, rowID=513114, timestamp="2017-05-18T20:40") - SetBit(frame="language", columnID=91720568, rowID=5) - SetBit(frame="language", columnID=95122322, rowID=17) - ' -``` diff --git a/docs/installation.md b/docs/installation.md index 5a0ea3086..a0632c34c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz + tar xfz pilosa-v0.9.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -134,19 +134,25 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: - * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) 2. Clone the repo: ``` - go get -d github.com/pilosa/pilosa + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa): +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` @@ -203,7 +209,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) docker version ``` -If you don't see the server listed, start the Docker application. + If you don't see the server listed, start the Docker application. 3. Pull the official Pilosa image from Docker Hub: ``` @@ -228,19 +234,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-linux-amd64.tar.gz + tar xfz pilosa-v0.9.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -287,19 +293,26 @@ There are three ways to install Pilosa on Linux: download the binary (recommende #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: - * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). * [Git](https://git-scm.com/) 2. Clone the repo: ``` - go get -d github.com/pilosa/pilosa + mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ + git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo: +3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps + make generate-statik make install ``` diff --git a/docs/introduction.md b/docs/introduction.md index d499c8c0a..d2ebb4662 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w "What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface. -Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. +Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. diff --git a/docs/pdk.md b/docs/pdk.md index 8c02dc279..4d1e599ad 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -2,66 +2,72 @@ title = "PDK" weight = 11 nav = [ + "Examples and Executables", "Library", - "Examples", ] +++ ## PDK -The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly. +The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively. + +### Examples and Executables +Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here. + +#### Kafka +`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the +Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each +record from Kafka is assigned a Pilosa column, and each value in a record is +assigned a row or field. Frame and field names are built from the "path" through +the record to arrive at that field. For example: + +```json +{ + "name": "jill", + "favorite_foods": ["corn chips", "chipotle dip"], + "location": { + "city": "Austin", + "state": "Texas", + "latitude": 3754, + "longitude": 4526 + }, + "active": true, + "age": 27 +} +``` + +This JSON object would result in the following Pilosa schema: + +| Name | Field | Type | Size/Min | Max | +|----------------|-----------|--------|----------|------------| +| name | | ranked | 100000 | | +| favorite_foods | | ranked | 100000 | | +| default | | Ranked | 100000 | | +| | age | int | 0 | 2147483647 | +| location | | ranked | 1000 | | +| | latitude | int | 0 | 2147483647 | +| | longitude | int | 0 | 2147483647 | +| location-city | | ranked | 100000 | | +| location-state | | ranked | 100000 | | + +All frames are created as ranked frames by default, and fields are created with +a minmum size of zero and a fixed maximum of 2147483647. Fields at the top level +are created in the default frame. Frames are a dash-separated concatenation of +all key values in the path - you can see this with frames like location-city. + + +Most the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts, +kafka topics, kafka group, etc.), but there are a few options that give some +control over the way data is indexed, and ingestion performance. + +* `--batch-size`: The batch size control how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. +* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state". +* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it. +* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used. +* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb. -The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary. ### Library -#### Mapping +For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation. -Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts: - -* an enumeration of field names -* a list of parsers that are used to parse strings in the CSV to values -* a list of commonly used, named, mapper functions -* a list of ParserMappers - objects that encapsulate all of the work related to a single frame. - -This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK. - -With this definition available, the PDK tool can run the import, which consists of these steps: - -- create the index -- create all frames -- for each CSV file, read all rows -- for each CSV record: - - generate a columnID - - apply all ParserMappers, generating a list of (frame, ID) pairs - - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) - -The process is summarized in this flowchart: - -![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg) - - -Some of the simple mapper functions available with PDK include: - -* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year. -* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11]. -* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6]. -* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23]. -* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins. -* BoolMapper: Maps a boolean value to the range [0, 1]. -* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain. -* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small. -* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets. -* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket. -* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example. -* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper. - -### Examples - -Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help ` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality. - - diff --git a/docs/query-language.md b/docs/query-language.md index f28257fe6..7d9f22886 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -13,7 +13,7 @@ nav = [ ### Overview -This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: +This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: ``` {"results":[...]} @@ -21,35 +21,34 @@ This section will provide a detailed reference and examples for the Pilosa Query There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type. -The default row label is `rowID`, and the default column label is `columnID`. Changing these defaults is deprecated and this feature will be removed in a future release. - #### Conventions * Angle Brackets `<>` denote required arguments * Square Brackets `[]` denote optional arguments -* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`) +* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`) ##### Examples -Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. +Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. -The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: -``` +The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would: +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' + -d 'SetBit(frame="stargazer", col=10, row=1)' +``` +``` response +{"results":[true]} ``` #### Arguments and Types -* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. -* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated. -* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated. +* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range` +* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -59,7 +58,7 @@ curl localhost:10101/index/repository/query \ **Spec:** ``` -SetBit(, , , +SetBit(, , , [timestamp=TIMESTAMP]) ``` @@ -77,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin **Examples:** ``` -SetBit(frame="stargazer", repo_id=10, rowID=1) +SetBit(frame="stargazer", col=10, row=1) ``` This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10. SetBit also supports providing a timestamp. To write the date that a user starred a repository. ``` -SetBit(frame="stargazer", repo_id=10, rowID=1, timestamp="2016-01-01T00:00") +SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") ``` Setting multiple bits in a single request: ``` -SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2) +SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2) ``` #### SetRowAttrs **Spec:** ``` -SetRowAttrs(, , +SetRowAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -112,13 +111,13 @@ SetRowAttrs queries always return `null` upon success. **Examples:** ``` -SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true) +SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true) ``` -Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. ``` -SetRowAttrs(frame="stargazer", rowID=10, username=null) +SetRowAttrs(frame="stargazer", row=10, username=null) ``` Delete username value for user 10. @@ -128,7 +127,7 @@ Delete username value for user 10. **Spec:** ``` -SetColumnAttrs(, , +SetColumnAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -144,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu **Examples:** ``` -SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) +SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`. ``` -SetColumnAttrs(columnID=10, url=null) +SetColumnAttrs(col=10, url=null) ``` Delete url value for repo 10. @@ -161,8 +160,8 @@ Delete url value for repo 10. **Spec:** ``` -SetBit(, , , - [timestamp=TIMESTAMP]) +ClearBit(, , , + [timestamp=TIMESTAMP]) ``` **Description:** @@ -178,10 +177,34 @@ A return value of `false` indicates that the bit was already set to 0 and nothin **Examples:** ``` -ClearBit(frame="stargazer", columnID=10, rowID=1) +ClearBit(frame="stargazer", col=10, row=1) ``` -Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame. +Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame. + + +#### SetFieldValue + +**Spec:** + +``` +SetFieldValue(, , ) +``` + +**Description:** + +`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`. + +**Result Type:** null + +SetFieldValue returns `null` upon success. + +**Examples:** + +Set the number of pull requests of repository 10. +``` +SetFieldValue(col=10, frame="stats", pullrequests=2) +``` ### Read Operations @@ -191,12 +214,12 @@ Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer fr **Spec:** ``` -Bitmap(, ( | =UINT)) +Bitmap(, ( | =UINT)) ``` **Description:** -`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. +`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column. **Result Type:** object with attrs and bits. @@ -206,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` Query all repositories that user 1 has starred. ``` -Bitmap(frame="stargazer", rowID=1) +Bitmap(frame="stargazer", row=1) ``` Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` @@ -263,7 +286,7 @@ attrs will always be empty Query repositories which have been starred by two users. ``` -Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) ``` Returns `{"attrs":{},"bits":[10]}`. @@ -290,7 +313,7 @@ attrs will always be empty Query repositories which have been starred by one user and not another. ``` -Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2)) +Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2)) ``` Return `{"results":[{"attrs":{},"bits":[20]}]}` @@ -298,13 +321,41 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}` * bits are repositories that were starred by user 1 BUT NOT user 2 ``` -Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1)) +Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1)) ``` Return `{"attrs":{},"bits":[30]}` * Bits are repositories that were starred by user 2 BUT NOT user 1 +#### Xor + +**Spec:** + +``` +Xor(, [BITMAP_CALL ...]) +``` + +**Description:** + +Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query repositories which have been starred by two users. + +``` +Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) +``` + +Returns `{"attrs":{},"bits":[30]}`. + +* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) + #### Count **Spec:** @@ -322,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in. Query the number of repositories to which a user has contributed. ``` -Count(Bitmap(frame="stargazer", rowID=1)) +Count(Bitmap(frame="stargazer", row=1)) ``` Return `2` @@ -335,13 +386,12 @@ Return `2` ``` TopN([BITMAP_CALL], , [n=UINT], - [inverse=true], [, ]) + [, ]) ``` **Description:** Return the id and count of the top `n` bitmaps (by count of bits) in the frame. -`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse). The `field` and `filters` arguments work together to only return Bitmaps which have the attribute specified by `field` with one of the values specified in `filters`. @@ -349,6 +399,7 @@ have the attribute specified by `field` with one of the values specified in **Result Type:** array of key/count objects **Caveats:** + * Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order. * Frames with cache type lru will maintain an LRU (Least Recently Used) cache, thus a TopN() query on this type of frame will return bitmaps sorted in order of most recently set bit. * The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN() queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. @@ -367,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1} * count is amount of repositories * Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository. -``` -TopN(frame="stargazer", inverse=true) -``` - -Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]` - -* key is a repository ID -* count is amount of users -* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame. - ``` TopN(frame="stargazer", n=2) ``` @@ -386,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]` * Results are the top two users sorted by number of repositories they've starred in descending order. ``` -TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2) +TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2) ``` Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` @@ -398,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` **Spec:** ``` -Range(, , +Range(, , , ) ``` @@ -414,7 +455,7 @@ between the given `start` and `end` timestamps. When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range. ``` -Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00") ``` Returns `{{"attrs":{},"bits":[10]}` @@ -441,7 +482,7 @@ Returns bits that are true for the comparison operator. **Examples:** In our source data, commitactivity was counted over the last year. -The following greater-than Range query returns all repositories having more than 100 commits. +The following greater-than `Range` query returns all repositories having more than 100 commits. ``` Range(frame="stats", commitactivity > 100) @@ -471,6 +512,56 @@ Range(frame="stats", commitactivity >< [100, 200]) This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +#### Min + +**Spec:** + +``` +Min([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered. + +**Result Type:** object with the min and count of columns containing the min value. + +**Examples:** + +Query the size of all repositories. +``` +Min(frame="stats", field="diskusage") +``` + +Return `{"min":4,"count":2}` + +* Result is the smallest repository in kilobytes, plus the number of repositories of that size. + +#### Max + +**Spec:** + +``` +Max([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. + +**Result Type:** object with the max and count of columns containing the max value. + +**Examples:** + +Query the size of all repositories. +``` +Max(frame="stats", field="diskusage") +``` + +Return `{"max":88,"count":13}` + +* Result is the largest repository in kilobytes, plus the number of repositories of that size. + #### Sum **Spec:** @@ -481,7 +572,7 @@ Sum([BITMAP_CALL], , ) **Description:** -Returns the count and computed sum of all bitmap encoded integer values across the `field` in this `frame`. The optional Bitmap call filters the bits used in this computation. +Returns the count and computed sum of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. **Result Type:** object with the computed sum and count of the bitmap field. @@ -495,27 +586,3 @@ Sum(frame="stats", field="diskusage") Return `{"sum":10,"count":3}` * Result is the size of all repositories in kilobytes, plus the number of repositories. - - -#### SetFieldValue - -**Spec:** - -``` -SetFieldValue(, , ) -``` - -**Description:** - -`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`. - -**Result Type:** null - -SetFieldValue returns `null` upon success. - -**Examples:** - -Set the number of pull requests of repository 10. -``` -SetFieldValue(col=10, frame="stats", pullrequests=2) -``` diff --git a/docs/tutorials.md b/docs/tutorials.md index 9c19f3a67..39e345dbd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -10,6 +10,16 @@ nav = [ ## Tutorials +
+ +Some of our tutorials work better as standalone repos, since you can git clone the instructions, code, and data all at once. Officially supported tutorials are listed here.
+
+ + +
+ ### Setting Up a Secure Cluster #### Introduction @@ -20,7 +30,7 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma #### Installing Pilosa and Creating the Directory Structure -If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source). +If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source). After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line: ``` @@ -45,6 +55,7 @@ openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.cr ``` The command above creates two files in the current directory: + * `pilosa.local.crt` is the SSL certificate. * `pilosa.local.key` is the private key file which must be kept as secret. @@ -59,6 +70,8 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. +One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated. + Create `node1.config.toml` in the project directory and paste the following in it: ```toml @@ -68,7 +81,7 @@ data-dir = "node1_data" bind = "https://01.pilosa.local:10501" [cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] +coordinator = true [tls] certificate = "pilosa.local.crt" @@ -76,7 +89,7 @@ key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 15000 key = "pilosa.local.gossip32" ``` @@ -89,16 +102,13 @@ Create `node2.config.toml` in the project directory and paste the following in i data-dir = "node2_data" bind = "https://02.pilosa.local:10502" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 16000 key = "pilosa.local.gossip32" ``` @@ -111,26 +121,24 @@ Create `node3.config.toml` in the project directory and paste the following in i data-dir = "node3_data" bind = "https://03.pilosa.local:10503" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 17000 key = "pilosa.local.gossip32" ``` Here is some explanation of the configuration items: + * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. -* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. +* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. #### Final Touches Before Running the Cluster @@ -177,38 +185,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. -All nodes should be in the `UP` state: -``` -{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} +All nodes should be in the `NORMAL` state: +``` response +{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]} ``` #### Running Queries Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` +``` response +{} +``` This will create index `sample-index` with default options. Let's create the frame now: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' ``` +``` response +{} +``` We just created frame `sample-frame` with default options. Let's run a `SetBit` query: +``` request +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)' ``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' +``` response +{"results":[true]} ``` Confirm that the bit was indeed set: +``` request +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` response +{"results":[{"attrs":{},"bits":[100]}]} ``` The same response should be returned when querying other nodes in the cluster: +``` request +curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` -curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +``` response +{"results":[{"attrs":{},"bits":[100]}]} ``` #### What's Next? @@ -220,27 +243,22 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: -``` +``` request curl localhost:10101/index/patients \ -X POST ``` - -Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient. -``` -curl localhost:10101/index/patients/frame/measurements \ - -X POST \ - -d '{"options":{"rangeEnabled": true}}' +``` response +{} ``` In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. -``` +``` request curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ - "rangeEnabled": true, "fields": [ {"name": "age", "type": "int", "min": 0, "max": 120}, {"name": "weight", "type": "int", "min": 0, "max": 500}, @@ -248,16 +266,22 @@ curl localhost:10101/index/patients/frame/measurements \ ] }}' ``` +``` response +{} +``` If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' + -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)' +``` +``` response +{"results":[null]} ``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -282,39 +306,80 @@ pilosa import -i patients -f measurements --field age ages.csv Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Range(frame="measurements", age > 40)' ``` -You should get the following results: -``` +``` response {"results":[{"attrs":{},"bits":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(frame="measurements", field="age")' ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -``` +``` response {"results":[{"sum":377,"count":9}]} ``` +The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. -``` +You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' ``` -Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. -``` +``` response {"results":[{"sum":191,"count":3}]} ``` +Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. + +To find the minimum age of all patients, run a `Min` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(frame="measurements", field="age")' +``` +``` response +{"results":[{"min":19,"count":1}]} +``` +The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"min":57,"count":1}]} +``` + +To find the maximum age of all patients, run a `Max` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(frame="measurements", field="age")' +``` +``` response +{"results":[{"max":71,"count":1}]} +``` +The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"max":34,"count":1}]} +``` ### Storing Row and Column Attributes @@ -323,95 +388,102 @@ Notice in this case that the count is only `3` because of the `age > 40` filter Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data. First, create an index called `books` to use for this tutorial: -``` +``` request curl localhost:10101/index/books \ -X POST ``` +``` response +{} +``` Next, create a frame in the `books` index called `members` which will represent library members who have read books. -``` +``` request curl localhost:10101/index/books/frame/members \ -X POST \ -d '{}' ``` +``` response +{} +``` Now, let's add some books to our index. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) - SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972) - SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000) - SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) - SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' + -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(col=2, name="No Name in the Street", year=1972) + SetColumnAttrs(col=3, name="The Tipping Point", year=2000) + SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(col=5, name="The Forever War", year=2008)' +``` +``` response +{"results":[null,null,null,null,null]} ``` And add some members. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") - SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins") - SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks") - SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") - SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' + -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith") + SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins") + SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks") + SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez") + SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")' +``` +``` response +{"results":[null,null,null,null,null]} ``` At this point we can query one of the `member` records by querying that row. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' -``` -You should get the following result set: + -d 'Bitmap(frame="members", row=10002)' ``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetBit(frame="members", rowID=10001, columnID=3) - SetBit(frame="members", rowID=10001, columnID=5) - - SetBit(frame="members", rowID=10002, columnID=1) - SetBit(frame="members", rowID=10002, columnID=2) - SetBit(frame="members", rowID=10002, columnID=4) - - SetBit(frame="members", rowID=10003, columnID=3) - - SetBit(frame="members", rowID=10004, columnID=4) - SetBit(frame="members", rowID=10004, columnID=5) - - SetBit(frame="members", rowID=10005, columnID=1) - SetBit(frame="members", rowID=10005, columnID=2) - SetBit(frame="members", rowID=10005, columnID=3) - SetBit(frame="members", rowID=10005, columnID=4) - SetBit(frame="members", rowID=10005, columnID=5)' + -d 'SetBit(frame="members", row=10001, col=3) + SetBit(frame="members", row=10001, col=5) + SetBit(frame="members", row=10002, col=1) + SetBit(frame="members", row=10002, col=2) + SetBit(frame="members", row=10002, col=4) + SetBit(frame="members", row=10003, col=3) + SetBit(frame="members", row=10004, col=4) + SetBit(frame="members", row=10004, col=5) + SetBit(frame="members", row=10005, col=1) + SetBit(frame="members", row=10005, col=2) + SetBit(frame="members", row=10005, col=3) + SetBit(frame="members", row=10005, col=4) + SetBit(frame="members", row=10005, col=5)' +``` +``` response +{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} ``` Now pull the record for `Sue Perkins` again. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' -``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. + -d 'Bitmap(frame="members", row=10002)' ``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} ``` +Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' -``` - -Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute. - + -d 'Bitmap(frame="members", row=10002)' ``` +``` response { "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], "columnAttrs":[ @@ -421,15 +493,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt ] } ``` +The `book` attributes are included in the result set at the `columnAttrs` attribute. Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members: -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' -``` - + -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))' ``` +``` response { "results":[{"attrs":{},"bits":[4]}], "columnAttrs":[ diff --git a/docs/webui.md b/docs/webui.md index cef8e570d..0f14934b4 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -9,18 +9,20 @@ nav = [ ## WebUI -The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101) +The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101). + This can be used for constructing queries and viewing the cluster status. ### Console -The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. +The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. Each query's result will be displayed in the Output section along with the query time. The Console will keep a record of each query and its result with the latest query on top. -![console](/img/docs/webui-console.png) +![webUI console screenshot](/img/docs/webui-console.png) +*WebUI console screenshot* In addition to standard PQL, the console supports a few special commands, prefixed with `:`. @@ -30,10 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference). +Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). -- `:create index timeQuantum=YM` -- `:create frame inverseEnabled=true cacheSize=10000` +- `:create frame cacheSize=10000` ### Cluster Admin diff --git a/event.go b/event.go new file mode 100644 index 000000000..5df69361b --- /dev/null +++ b/event.go @@ -0,0 +1,55 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +// NodeEventType are the types of events that can be sent from the +// ChannelEventDelegate. +type NodeEventType int + +const ( + NodeJoin NodeEventType = iota + NodeLeave + NodeUpdate +) + +// NodeEvent is a single event related to node activity in the cluster. +type NodeEvent struct { + Event NodeEventType + Node *Node +} + +// EventHandler is the interface for the pilosa object which knows how to +// handle broadcast messages. (Hint: this is implemented by pilosa.Server) +type EventHandler interface { + ReceiveEvent(e *NodeEvent) error +} + +// EventReceiver is the interface for the object which will listen for and +// decode broadcast messages before passing them to pilosa to handle. The +// implementation of this could be an http server which listens for messages, +// gets the protobuf payload, and then passes it to +// EventHandler.ReceiveMessage. +type EventReceiver interface { + // Start starts listening for broadcast messages - it should return + // immediately, spawning a goroutine if necessary. + Start(EventHandler) error +} + +type nopEventReceiver struct{} + +func (n *nopEventReceiver) Start(e EventHandler) error { return nil } + +// NopEventReceiver is a no-op implementation of the EventReceiver. +var NopEventReceiver = &nopEventReceiver{} diff --git a/executor.go b/executor.go index d12aeff1f..b6dc08d41 100644 --- a/executor.go +++ b/executor.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "errors" "fmt" "net/http" "sort" @@ -24,6 +23,7 @@ import ( "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) // DefaultFrame is the frame used if one is not specified. @@ -33,6 +33,9 @@ const ( // MinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. MinThreshold = 1 + + columnLabel = "col" + rowLabel = "row" ) // Executor recursively executes calls in a PQL query across all slices. @@ -40,8 +43,7 @@ type Executor struct { Holder *Holder // Local hostname & cluster configuration. - Scheme string - Host string + Node *Node Cluster *Cluster // Client used for remote requests. @@ -81,35 +83,32 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // MaxSlice can differ between inverse and standard views, so we need // to send queries to different slices based on orientation. var inverseSlices []uint64 - rowLabel := DefaultRowLabel - columnLabel := DefaultColumnLabel - // If slices aren't specified, then include all of them. - if len(slices) == 0 { - // Determine slices and inverseSlices for use in e.executeCall(). - if needsSlices { - // Round up the number of slices. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - maxSlice := idx.MaxSlice() - maxInverseSlice := idx.MaxInverseSlice() + // If slices are specified, then use that value for slices or + // inverseSlices. If slices aren't specified, then include all of them. + if len(slices) > 0 { + // For inverse queries, the values of `slices` provided to the Execute() method + // on the remote node actually represents inverseSlices. + inverseSlices = slices + } else if needsSlices { + // Round up the number of slices. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + maxSlice := idx.MaxSlice() + maxInverseSlice := idx.MaxInverseSlice() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) - } + // Generate a slices of all slices. + slices = make([]uint64, maxSlice+1) + for i := range slices { + slices[i] = uint64(i) + } - // Generate a slices of all inverse slices. - inverseSlices = make([]uint64, maxInverseSlice+1) - for i := range inverseSlices { - inverseSlices[i] = uint64(i) - } - - // Fetch column label from index. - columnLabel = idx.ColumnLabel() + // Generate a slices of all inverse slices. + inverseSlices = make([]uint64, maxInverseSlice+1) + for i := range inverseSlices { + inverseSlices[i] = uint64(i) } } @@ -121,7 +120,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - if call.SupportsInverse() && needsSlices { // Fetch frame & row label based on argument. frame, _ := call.Args["frame"].(string) @@ -132,7 +130,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic if f == nil { return nil, ErrFrameNotFound } - rowLabel = f.RowLabel() // If this call is to an inverse frame send to a different list of slices. if call.IsInverse(rowLabel, columnLabel) { @@ -151,9 +148,8 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // executeCall executes a call. func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { - if err := e.validateCallArgs(c); err != nil { - return nil, err + return nil, errors.Wrap(err, "validating args") } indexTag := fmt.Sprintf("index:%s", index) // Special handling for mutation and top-n calls. @@ -161,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeSum(ctx, index, c, slices, opt) + case "Min": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMin(ctx, index, c, slices, opt) + case "Max": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -203,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return SumCount{}, errors.New("Sum(): frame required") + return ValCount{}, errors.New("Sum(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return SumCount{}, errors.New("Sum(): field required") + return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { - return SumCount{}, errors.New("Sum() only accepts a single bitmap input") + return ValCount{}, errors.New("Sum() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -221,18 +223,88 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(SumCount) - return other.Add(v.(SumCount)) + other, _ := prev.(ValCount) + return other.Add(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return SumCount{}, err + return ValCount{}, err } - other, _ := result.(SumCount) + other, _ := result.(ValCount) if other.Count == 0 { - return SumCount{}, nil + return ValCount{}, nil + } + return other, nil +} + +// executeFieldMin executes a Min() call. +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return ValCount{}, errors.New("Min(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Min(): field required") + } + + if len(c.Children) > 1 { + return ValCount{}, errors.New("Min() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMinSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(ValCount) + return other.Smaller(v.(ValCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return ValCount{}, err + } + other, _ := result.(ValCount) + + if other.Count == 0 { + return ValCount{}, nil + } + return other, nil +} + +// executeFieldMax executes a Max() call. +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return ValCount{}, errors.New("Max(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return ValCount{}, errors.New("Max(): field required") + } + + if len(c.Children) > 1 { + return ValCount{}, errors.New("Max() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMaxSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(ValCount) + return other.Larger(v.(ValCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return ValCount{}, err + } + other, _ := result.(ValCount) + + if other.Count == 0 { + return ValCount{}, nil } return other, nil } @@ -269,11 +341,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { idx := e.Holder.Index(index) if idx != nil { - columnLabel := idx.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting column attrs") } bm.Attrs = attrs } else if err != nil { @@ -281,14 +352,13 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else { frame, _ := c.Args["frame"].(string) if fr := idx.Frame(frame); fr != nil { - rowLabel := fr.RowLabel() rowID, _, err := c.UintArg(rowLabel) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting row") } attrs, err := fr.RowAttrStore().Attrs(rowID) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting row attrs") } bm.Attrs = attrs } @@ -324,13 +394,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice executes calculates the sum & count for fields on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { +// executeSumCountSlice calculates the sum and count for fields on a slice. +func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return SumCount{}, err + return ValCount{}, errors.Wrap(err, "executing bitmap call") } filter = bm } @@ -340,29 +410,107 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return SumCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return SumCount{}, nil + return ValCount{}, nil } - view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) - if view == nil { - return SumCount{}, nil + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil } - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { - return SumCount{}, err + return ValCount{}, errors.Wrap(err, "computing sum") } - return SumCount{ - Sum: int64(vsum) + (int64(vcount) * field.Min), + return ValCount{ + Val: int64(vsum) + (int64(vcount) * field.Min), Count: int64(vcount), }, nil } +// executeFieldMinSlice calculates the min for fields on a slice. +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return ValCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return ValCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil + } + + fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + if err != nil { + return ValCount{}, err + } + return ValCount{ + Val: int64(fmin) + field.Min, + Count: int64(fcount), + }, nil +} + +// executeFieldMaxSlice calculates the max for fields on a slice. +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return ValCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return ValCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return ValCount{}, nil + } + + fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + if err != nil { + return ValCount{}, err + } + return ValCount{ + Val: int64(fmax) + field.Min, + Count: int64(fcount), + }, nil +} + // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. @@ -379,7 +527,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // Execute original query. pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "finding top results") } // If this call is against specific ids, or we didn't get results, @@ -396,7 +544,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "retrieving full counts") } if n != 0 && int(n) < len(trimmedList) { @@ -526,7 +674,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Fetch frame & row label based on argument. frame, _ := c.Args["frame"].(string) @@ -537,7 +684,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(rowLabel) @@ -607,14 +753,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C if idx == nil { return nil, ErrIndexNotFound } - columnLabel := idx.ColumnLabel() // Retrieve base frame. f := idx.Frame(frame) if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() // Read row & column id. columnID, columnOK, err := c.UintArg(columnLabel) @@ -739,7 +883,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * predicates, err := cond.IntSliceValue() if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting condition value") } // Only support two integers for the between operation. @@ -801,7 +945,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return NewBitmap(), nil } - // LT[E] and GT[E] should return all not-null if selected range fully encompases valid field range. + // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { return frag.FieldNotNull(field.BitDepth()) @@ -905,10 +1049,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(rowLabel) if err != nil { @@ -955,9 +1095,9 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. - if node.Host == e.Host { + if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) if err != nil { return false, err @@ -999,10 +1139,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFrameNotFound } - // Retrieve labels. - columnLabel := idx.ColumnLabel() - rowLabel := f.RowLabel() - // Read fields using labels. rowID, ok, err := c.UintArg(rowLabel) if err != nil { @@ -1060,9 +1196,9 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. - if node.Host == e.Host { + if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) if err != nil { return false, err @@ -1094,13 +1230,6 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq return errors.New("SetFieldValue() frame required") } - // Retrieve column label. - idx := e.Holder.Index(index) - if idx == nil { - return ErrIndexNotFound - } - columnLabel := idx.ColumnLabel() - // Retrieve frame. frame := e.Holder.Frame(index, frameName) if frame == nil { @@ -1141,7 +1270,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1172,7 +1301,6 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if frame == nil { return ErrFrameNotFound } - rowLabel := frame.RowLabel() // Parse labels. rowID, ok, err := c.UintArg(rowLabel) @@ -1199,7 +1327,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1233,7 +1361,6 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if f == nil { return nil, ErrFrameNotFound } - rowLabel := f.RowLabel() rowID, ok, err := c.UintArg(rowLabel) if err != nil { @@ -1286,7 +1413,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1314,28 +1441,18 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return ErrIndexNotFound } - var colName string - id, okID, errID := c.UintArg("id") - if errID != nil || !okID { - // Retrieve columnLabel - columnLabel := idx.columnLabel - col, okCol, errCol := c.UintArg(columnLabel) - if errCol != nil || !okCol { - return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) - } - id = col - colName = columnLabel - } else { - colName = "id" + col, okCol, errCol := c.UintArg(columnLabel) + if errCol != nil || !okCol { + return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, colName) + delete(attrs, columnLabel) delete(attrs, "frame") // Set attributes. - if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil { + if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { return err } idx.Stats.Count("SetProfileAttrs", 1, 1.0) @@ -1345,7 +1462,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p } // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host) + nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { @@ -1372,13 +1489,8 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * Slices: slices, Remote: true, } - uri, err := NewURIFromAddress(node.Host) - if err != nil { - return nil, err - } - uri.SetScheme(node.Scheme) - ctx = context.WithValue(ctx, "uri", uri) - pb, err := e.client.ExecuteQuery(ctx, index, pbreq) + + pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq) if err != nil { return nil, err } @@ -1396,7 +1508,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * switch call.Name { case "Average", "Sum": - v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil + v, err = decodeValCount(pb.Results[i].GetValCount()), nil case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": @@ -1426,7 +1538,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop @@ -1456,12 +1568,12 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.NodeByHost(e.Host)} + nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} } // Start mapping across all primary owners. if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { - return nil, err + return nil, errors.Wrap(err, "starting mapper") } // Iterate over all map responses and reduce. @@ -1512,7 +1624,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod resp := mapResponse{node: n, slices: nodeSlices} // Send local slices to mapper, otherwise remote exec. - if n.Host == e.Host { + if n.ID == e.Node.ID { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) @@ -1634,29 +1746,51 @@ func needsSlices(calls []*pql.Call) bool { return false } -// SumCount represents a grouping of sum & count for Sum() and Average() calls. -type SumCount struct { - Sum int64 `json:"sum"` +// ValCount represents a grouping of sum & count for Sum() and Average() calls. +type ValCount struct { + Val int64 `json:"value"` Count int64 `json:"count"` } -func (sc *SumCount) Add(other SumCount) SumCount { - return SumCount{ - Sum: sc.Sum + other.Sum, - Count: sc.Count + other.Count, +func (vc *ValCount) Add(other ValCount) ValCount { + return ValCount{ + Val: vc.Val + other.Val, + Count: vc.Count + other.Count, } } -func encodeSumCount(sc SumCount) *internal.SumCount { - return &internal.SumCount{ - Sum: sc.Sum, - Count: sc.Count, +func encodeValCount(vc ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, } } -func decodeSumCount(pb *internal.SumCount) SumCount { - return SumCount{ - Sum: pb.Sum, +func decodeValCount(pb *internal.ValCount) ValCount { + return ValCount{ + Val: pb.Val, Count: pb.Count, } } + +// Smaller returns the smaller of the two ValCounts. +func (vc *ValCount) Smaller(other ValCount) ValCount { + if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { + return other + } + return ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +// Larger returns the larger of the two ValCounts. +func (vc *ValCount) Larger(other ValCount) ValCount { + if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { + return other + } + return ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} diff --git a/executor_test.go b/executor_test.go index dd8725298..0fd8a925b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit bits. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected bits: %+v", bits) @@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -122,7 +122,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -154,7 +154,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -184,7 +184,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -217,7 +217,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -233,7 +233,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, @@ -293,9 +292,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Set field values. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { t.Fatal(err) } @@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, }, @@ -340,28 +338,28 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Run("ErrFrameRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldRequired", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnFieldValue", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidFieldValueType", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { t.Fatalf("unexpected error: %s", err) } }) @@ -384,16 +382,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -419,15 +417,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=1) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, rowID=10, columnID=0) - SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, rowID=0, columnID=0) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=1) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, row=10, col=0) + SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, row=0, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -543,7 +541,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -590,7 +588,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -599,6 +597,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } } +// Ensure Min() and Max() queries can be executed. +func TestExecutor_Execute_MinMax(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=3) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=1, col=1) + SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + + SetFieldValue(frame=f, foo=20, col=0) + SetFieldValue(frame=f, foo=-5, col=1) + SetFieldValue(frame=f, foo=-5, col=2) + SetFieldValue(frame=f, foo=10, col=3) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) +} + // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() @@ -611,7 +701,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -621,7 +710,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -630,15 +718,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) + SetFieldValue(frame=f, foo=20, bar=2000, col=0) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } @@ -646,15 +734,15 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -679,22 +767,22 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, rowID=1, columnID=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, rowID=1, columnID=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, rowID=1, columnID=7, timestamp="2002-01-01T02:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") + SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") + SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") + SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, rowID=1, columnID=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, rowID=10, columnID=2, timestamp="2001-01-01T00:00") + SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") + SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") + SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -703,7 +791,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("Inverse", func(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { t.Fatalf("unexpected bits: %+v", bits) @@ -723,7 +811,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -733,7 +820,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -742,7 +828,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, }, @@ -751,17 +836,17 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, rowID=0, columnID=0) - SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) - SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, columnID=0) - SetFieldValue(frame=edge, foo=100, columnID=0) - SetFieldValue(frame=edge, foo=-100, columnID=1) + SetFieldValue(frame=f, foo=20, bar=2000, col=50) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, col=0) + SetFieldValue(frame=edge, foo=100, col=0) + SetFieldValue(frame=edge, foo=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } @@ -897,14 +982,18 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Scheme = "http" - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", rowID=10)` { + } else if query.String() != `Bitmap(frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -923,11 +1012,11 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) @@ -941,7 +1030,13 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to return a count. s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -951,12 +1046,12 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -971,14 +1066,20 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments. var remoteCalled bool s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` { + } else if query.String() != `SetBit(col=2, frame="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -988,7 +1089,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -996,7 +1097,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -1017,14 +1118,20 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments. var remoteCalled bool s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1034,7 +1141,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1044,7 +1151,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } @@ -1064,7 +1171,13 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create secondary server and update second cluster node. s := test.NewServer() defer s.Close() - c.Nodes[1].Host = s.Host() + + uri, err := pilosa.NewURIFromAddress(s.Host()) + if err != nil { + t.Fatal(err) + } + + c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int @@ -1102,7 +1215,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) @@ -1141,11 +1254,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=10)"), nil, nil) + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', columnID=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1158,11 +1271,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=20)"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(columnID=20, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/filesystem.go b/filesystem.go new file mode 100644 index 000000000..5664f0987 --- /dev/null +++ b/filesystem.go @@ -0,0 +1,42 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "net/http" +) + +// Ensure nopFileSystem implements interface. +var _ FileSystem = &nopFileSystem{} + +// FileSystem represents an interface for a WebUI file system. +type FileSystem interface { + New() (http.FileSystem, error) +} + +func init() { + NopFileSystem = &nopFileSystem{} +} + +// NopFileSystem represents a FileSystem that returns an error if called. +var NopFileSystem FileSystem + +type nopFileSystem struct{} + +// New is a no-op implementation of FileSystem New method. +func (n *nopFileSystem) New() (http.FileSystem, error) { + return nil, fmt.Errorf("file system not implemented") +} diff --git a/fragment.go b/fragment.go index 0da6043cf..97b756213 100644 --- a/fragment.go +++ b/fragment.go @@ -20,14 +20,12 @@ import ( "bytes" "container/heap" "context" - "crypto/sha1" "encoding/binary" "errors" "fmt" "hash" "io" "io/ioutil" - "log" "net/http" "os" "sort" @@ -36,6 +34,8 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash" + "math" "github.com/gogo/protobuf/proto" @@ -102,12 +102,12 @@ type Fragment struct { // so that they can be mmapped and heap utilization can be kept low. MaxOpN int - // Writer used for out-of-band log entries. - LogOutput io.Writer + // Logger used for out-of-band log entries. + Logger Logger // Row attribute storage. // This is set by the parent frame unless overridden for testing. - RowAttrStore *AttrStore + RowAttrStore AttrStore stats StatsClient } @@ -123,8 +123,8 @@ func NewFragment(path, index, frame, view string, slice uint64) *Fragment { CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, - LogOutput: ioutil.Discard, - MaxOpN: DefaultFragmentMaxOpN, + Logger: NopLogger, + MaxOpN: DefaultFragmentMaxOpN, stats: NopStatsClient, } @@ -255,6 +255,7 @@ func (f *Fragment) openCache() error { f.cache = NewLRUCache(f.CacheSize) case CacheTypeNone: f.cache = NewNopCache() + return nil default: return ErrInvalidCacheType } @@ -271,7 +272,7 @@ func (f *Fragment) openCache() error { // Unmarshal cache data. var pb internal.Cache if err := proto.Unmarshal(buf, &pb); err != nil { - f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) + f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err) return nil } @@ -296,13 +297,13 @@ func (f *Fragment) Close() error { func (f *Fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { - f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) return err } // Close underlying storage. if err := f.closeStorage(); err != nil { - f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) + f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) return err } @@ -341,9 +342,6 @@ func (f *Fragment) closeStorage() error { return nil } -// logger returns a logger instance for the fragment.nt. -func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) } - // Row returns a row by ID. func (f *Fragment) Row(rowID uint64) *Bitmap { f.mu.Lock() @@ -616,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e return sum, count, nil } +// FieldMin returns the min of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := consider.Difference(row) + count = x.Count() + if count > 0 { + consider = x + } else { + min += (1 << ii) + if ii == 0 { + count = consider.Count() + } + } + } + + return min, count, nil +} + +// FieldMax returns the max of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := row.Intersect(consider) + count = x.Count() + if count > 0 { + max += (1 << ii) + consider = x + } else if ii == 0 { + count = consider.Count() + } + } + + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { @@ -1020,7 +1082,7 @@ type TopOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { - h := sha1.New() + h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) } @@ -1383,18 +1445,17 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } -func track(start time.Time, message string, stats StatsClient, logger *log.Logger) { +func track(start time.Time, message string, stats StatsClient, logger Logger) { elapsed := time.Since(start) logger.Printf("%s took %s", message, elapsed) stats.Histogram("snapshot", elapsed.Seconds(), 1.0) } func (f *Fragment) snapshot() error { - logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) start := time.Now() - defer track(start, completeMessage, f.stats, logger) + defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt @@ -1454,6 +1515,10 @@ func (f *Fragment) flushCache() error { return nil } + if f.CacheType == CacheTypeNone { + return nil + } + // Retrieve a list of row ids from the cache. ids := f.cache.IDs() @@ -1660,7 +1725,7 @@ type blockHasher struct { func newBlockHasher() blockHasher { return blockHasher{ blockID: -1, - hash: sha1.New(), + hash: xxhash.New(), } } func (h *blockHasher) Reset() { @@ -1680,7 +1745,7 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Host string + Node *Node Cluster *Cluster RemoteClient *http.Client @@ -1701,7 +1766,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. - nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) if len(nodes) == 1 { return nil } @@ -1710,17 +1775,14 @@ func (s *FragmentSyncer) SyncFragment() error { blockSets := make([][]FragmentBlock, 0, len(nodes)) for _, node := range nodes { // Read local blocks. - if node.Host == s.Host { + if node.ID == s.Node.ID { b := s.Fragment.Blocks() blockSets = append(blockSets, b) continue } // Retrieve remote blocks. - client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) - if err != nil { - return err - } + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return err @@ -1786,8 +1848,8 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet var clients []InternalClient - for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { - if s.Host == node.Host { + for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { + if s.Node.ID == node.ID { continue } @@ -1796,10 +1858,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) - if err != nil { - return err - } + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) clients = append(clients, client) // Only sync the standard block. @@ -1837,15 +1896,19 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. total := len(set.ColumnIDs) + len(clear.ColumnIDs) - buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest)))) + maxWrites := s.Cluster.MaxWritesPerRequest + if maxWrites <= 0 { + maxWrites = 5000 + } + buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } @@ -1861,7 +1924,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].ExecuteQuery(context.Background(), f.Index(), queryRequest) + _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) if err != nil { return err } diff --git a/fragment_test.go b/fragment_test.go index 754bd8dd1..2b0c363b4 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -163,7 +163,7 @@ func TestFragment_SetFieldValue(t *testing.T) { t.Fatal("expected change") } - // Non-existant value. + // Non-existent value. if value, exists, err := f.FieldValue(100, 11); err != nil { t.Fatal(err) } else if value != 0 { @@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) { }) } +// Ensure a fragment can find the min and max of field values. +func TestFragment_FieldMinMax(t *testing.T) { + const bitDepth = 16 + + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 0, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if min != test.exp { + t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 2818, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if max != test.exp { + t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) +} + // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { const bitDepth = 16 @@ -702,7 +775,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { Fragment: frag, RowAttrStore: test.MustOpenAttrStore(), } - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore if err := f.Open(); err != nil { panic(err) } @@ -1075,25 +1148,3 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } - -func BenchmarkFragment_Snapshot(b *testing.B) { - if *FragmentPath == "" { - b.Skip("no fragment specified") - } - - b.ReportAllocs() - // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) - if err := f.Open(); err != nil { - b.Fatal(err) - } - defer f.Close() - b.ResetTimer() - // Reset timer and execute benchmark. - for i := 0; i < b.N; i++ { - err := f.Snapshot() - if err != nil { - b.Fatalf("unexpected count (reopen): %s", err) - } - } -} diff --git a/frame.go b/frame.go index 857c1b94c..3de3f3613 100644 --- a/frame.go +++ b/frame.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -32,10 +31,8 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "rowID" DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false - DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -43,31 +40,27 @@ const ( // Frame represents a container for views. type Frame struct { - mu sync.RWMutex - path string - index string - name string - timeQuantum TimeQuantum - schema *FrameSchema + mu sync.RWMutex + path string + index string + name string views map[string]*View // Row attribute storage and cache - rowAttrStore *AttrStore + rowAttrStore AttrStore broadcaster Broadcaster Stats StatsClient - // Frame settings. - rowLabel string - cacheType string + // Frame options. inverseEnabled bool - rangeEnabled bool + cacheType string + cacheSize uint32 + timeQuantum TimeQuantum + fields []*Field - // Cache size for ranked frames - cacheSize uint32 - - LogOutput io.Writer + Logger Logger } // NewFrame returns a new instance of frame. @@ -78,24 +71,24 @@ func NewFrame(path, index, name string) (*Frame, error) { } return &Frame{ - path: path, - index: index, - name: name, - schema: &FrameSchema{}, + path: path, + index: index, + name: name, - views: make(map[string]*View), - rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), + views: make(map[string]*View), + + rowAttrStore: NopAttrStore, broadcaster: NopBroadcaster, Stats: NopStatsClient, - rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, - rangeEnabled: DefaultRangeEnabled, cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, + //timeQuantum + //fields - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -109,7 +102,7 @@ func (f *Frame) Index() string { return f.index } func (f *Frame) Path() string { return f.path } // RowAttrStore returns the attribute storage. -func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore } +func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. func (f *Frame) MaxSlice() uint64 { @@ -139,39 +132,6 @@ func (f *Frame) MaxInverseSlice() uint64 { return view.MaxSlice() } -// SetRowLabel sets the row labels. Persists to meta file on update. -func (f *Frame) SetRowLabel(v string) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || f.rowLabel == v { - return nil - } - - // Make sure rowLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - f.rowLabel = v - if err := f.saveMeta(); err != nil { - return err - } - - return nil -} - -// RowLabel returns the row label. -func (f *Frame) RowLabel() string { - f.mu.RLock() - v := f.rowLabel - f.mu.RUnlock() - return v -} - // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { return f.cacheType @@ -182,11 +142,6 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } -// RangeEnabled returns true if range fields can be stored on this frame. -func (f *Frame) RangeEnabled() bool { - return f.rangeEnabled -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -224,13 +179,11 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ - RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, - RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, - Fields: f.schema.Fields, + Fields: f.fields, } } @@ -244,8 +197,6 @@ func (f *Frame) Open() error { if err := f.loadMeta(); err != nil { return err - } else if err := f.loadSchema(); err != nil { - return err } if err := f.openViews(); err != nil { @@ -304,12 +255,11 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.timeQuantum = "" - f.rowLabel = DefaultRowLabel - f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled - f.rangeEnabled = DefaultRangeEnabled + f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize + f.timeQuantum = "" + //f.fields return nil } else if err != nil { return err @@ -320,17 +270,14 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.rangeEnabled = pb.RangeEnabled - f.cacheSize = pb.CacheSize - - // Copy cache type. f.cacheType = pb.CacheType if f.cacheType == "" { f.cacheType = DefaultCacheType } + f.cacheSize = pb.CacheSize + f.timeQuantum = TimeQuantum(pb.TimeQuantum) + f.fields = decodeFields(pb.Fields) return nil } @@ -352,35 +299,6 @@ func (f *Frame) saveMeta() error { return nil } -// loadSchema reads the schema for the frame. -func (f *Frame) loadSchema() error { - buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema")) - if os.IsNotExist(err) { - f.schema = &FrameSchema{} - return nil - } else if err != nil { - return err - } - - var pb internal.FrameSchema - if err := proto.Unmarshal(buf, &pb); err != nil { - return err - } - f.schema = decodeFrameSchema(&pb) - - return nil -} - -// saveSchema writes the current schema to disk. -func (f *Frame) saveSchema() error { - if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil { - return err - } - return nil -} - // Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() @@ -402,16 +320,11 @@ func (f *Frame) Close() error { return nil } -// Schema returns the frame's current schema. -func (f *Frame) Schema() *FrameSchema { +// Field returns a field by name. +func (f *Frame) Field(name string) *Field { f.mu.RLock() defer f.mu.RUnlock() - return f.schema -} - -// Field returns a field from the schema by name. -func (f *Frame) Field(name string) *Field { - for _, field := range f.Schema().Fields { + for _, field := range f.fields { if field.Name == name { return field } @@ -419,42 +332,66 @@ func (f *Frame) Field(name string) *Field { return nil } -// CreateField creates a new field on the schema. +// Fields returns the fields on the frame. +func (f *Frame) Fields() []*Field { + f.mu.RLock() + defer f.mu.RUnlock() + return f.fields +} + +// HasField returns true if a field exists on the frame. +func (f *Frame) HasField(name string) bool { + for _, fld := range f.fields { + if fld.Name == name { + return true + } + } + return false +} + +// CreateField creates a new field on the frame. func (f *Frame) CreateField(field *Field) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - - // Copy schema and append field. - schema := f.schema.Clone() - if err := schema.AddField(field); err != nil { + // Append field. + if err := f.addField(field); err != nil { return err } - f.schema = schema - f.saveSchema() + f.saveMeta() + return nil +} + +// addField adds a single field to fields. +func (f *Frame) addField(field *Field) error { + if err := ValidateField(field); err != nil { + return err + } else if f.HasField(field.Name) { + return ErrFieldExists + } + + // Add field to list. + f.fields = append(f.fields, field) + + // Sort fields by name. + sort.Slice(f.fields, func(i, j int) bool { + return f.fields[i].Name < f.fields[j].Name + }) + return nil } // GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() (*FrameSchema, error) { +func (f *Frame) GetFields() ([]*Field, error) { f.mu.RLock() defer f.mu.RUnlock() - // Ensure the frame supports fields. - if !f.RangeEnabled() { - return nil, ErrFrameFieldsNotAllowed - } - - err := f.loadSchema() + err := f.loadMeta() if err != nil { return nil, err } - return f.schema, nil + return f.fields, nil } // DeleteField deletes an existing field on the schema. @@ -462,17 +399,10 @@ func (f *Frame) DeleteField(name string) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - - // Copy schema and remove field. - schema := f.schema.Clone() - if err := schema.DeleteField(name); err != nil { + // Remove field. + if err := f.deleteField(name); err != nil { return err } - f.schema = schema // Remove views. viewName := ViewFieldPrefix + name @@ -489,6 +419,18 @@ func (f *Frame) DeleteField(name string) error { return nil } +// deleteField removes a single field from fields. +func (f *Frame) deleteField(name string) error { + for i, field := range f.fields { + if field.Name == name { + copy(f.fields[i:], f.fields[i+1:]) + f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil + return nil + } + } + return ErrFieldNotFound +} + // TimeQuantum returns the time quantum for the frame. func (f *Frame) TimeQuantum() TimeQuantum { f.mu.Lock() @@ -543,6 +485,18 @@ func (f *Frame) Views() []*View { return other } +// viewNames returns a list of all views (as a string) in the frame. +func (f *Frame) viewNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + + other := make([]string, 0, len(f.views)) + for viewName, _ := range f.views { + other = append(other, viewName) + } + return other +} + // RecalculateCaches recalculates caches on every view in the frame. func (f *Frame) RecalculateCaches() { for _, view := range f.Views() { @@ -551,33 +505,60 @@ func (f *Frame) RecalculateCaches() { } // CreateViewIfNotExists returns the named view, creating it if necessary. +// Additionally, a CreateViewMessage is sent to the cluster. func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { + + view, created, err := f.createViewIfNotExistsBase(name) + if err != nil { + return nil, err + } + + if created { + // Broadcast view creation to the cluster. + err = f.broadcaster.SendSync( + &internal.CreateViewMessage{ + Index: f.index, + Frame: f.name, + View: name, + }) + if err != nil { + return nil, err + } + } + + return view, nil +} + +// createViewIfNotExistsBase returns the named view, creating it if necessary. +// The returned bool indicates whether the view was created or not. +func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { // Don't create inverse views if they are not enabled. if !f.InverseEnabled() && IsInverseView(name) { - return nil, ErrFrameInverseDisabled + return nil, false, ErrFrameInverseDisabled } f.mu.Lock() defer f.mu.Unlock() if view := f.views[name]; view != nil { - return view, nil + return view, false, nil } view := f.newView(f.ViewPath(name), name) + if err := view.Open(); err != nil { - return nil, err + return nil, false, err } view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view - return view, nil + return view, true, nil } func (f *Frame) newView(path, name string) *View { view := NewView(path, f.index, f.name, name, f.cacheSize) view.cacheType = f.cacheType - view.LogOutput = f.LogOutput + view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster @@ -756,6 +737,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } +// FieldMin returns the min for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + field.Min, int64(vcount), nil +} + +// FieldMax returns the max for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + field.Min, int64(vcount), nil +} + func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) @@ -813,7 +834,11 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Split import data by fragment. dataByFragment := make(map[importKey]importData) for i := range rowIDs { - rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i] + rowID, columnID := rowIDs[i], columnIDs[i] + var timestamp *time.Time + if len(timestamps) > i { + timestamp = timestamps[i] + } var standard, inverse []string if timestamp == nil { @@ -883,11 +908,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - // Verify that this frame is range-encoded. - if !f.RangeEnabled() { - return fmt.Errorf("Frame not RangeEnabled: %s", f.name) - } - viewName := ViewFieldPrefix + fieldName // Get the field so we know bitDepth. field := f.Field(fieldName) @@ -956,8 +976,9 @@ func encodeFrames(a []*Frame) []*internal.Frame { func encodeFrame(f *Frame) *internal.Frame { fo := f.options() return &internal.Frame{ - Name: f.name, - Meta: fo.Encode(), + Name: f.name, + Meta: fo.Encode(), + Views: f.viewNames(), } } @@ -969,8 +990,9 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FrameInfo represents schema information for a frame. type FrameInfo struct { - Name string `json:"name"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + Options FrameOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type frameInfoSlice []*FrameInfo @@ -981,9 +1003,8 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` - RangeEnabled bool `json:"rangeEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` @@ -992,10 +1013,15 @@ type FrameOptions struct { // Encode converts o into its internal representation. func (o *FrameOptions) Encode() *internal.FrameMeta { + return encodeFrameOptions(o) +} + +func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { + if o == nil { + return nil + } return &internal.FrameMeta{ - RowLabel: o.RowLabel, InverseEnabled: o.InverseEnabled, - RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), @@ -1003,74 +1029,16 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { } } -// FrameSchema represents the list of fields on a frame. -type FrameSchema struct { - Fields []*Field -} - -// Clone returns a copy of s. -func (s *FrameSchema) Clone() *FrameSchema { - other := &FrameSchema{Fields: make([]*Field, len(s.Fields))} - copy(other.Fields, s.Fields) - return other -} - -// HasField returns true if a field exists on the schema. -func (s *FrameSchema) HasField(name string) bool { - for _, f := range s.Fields { - if f.Name == name { - return true - } - } - return false -} - -// AddField adds a single field to the schema. -func (s *FrameSchema) AddField(field *Field) error { - if err := ValidateField(field); err != nil { - return err - } else if s.HasField(field.Name) { - return ErrFieldExists - } - - // Add field to list. - s.Fields = append(s.Fields, field) - - // Sort fields by name. - sort.Slice(s.Fields, func(i, j int) bool { - return s.Fields[i].Name < s.Fields[j].Name - }) - - return nil -} - -// DeleteField removes a single field from the schema. -func (s *FrameSchema) DeleteField(name string) error { - for i, field := range s.Fields { - if field.Name == name { - copy(s.Fields[i:], s.Fields[i+1:]) - s.Fields, s.Fields[len(s.Fields)-1] = s.Fields[:len(s.Fields)-1], nil - return nil - } - } - return ErrFieldNotFound -} - -func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema { - if schema == nil { +func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { + if options == nil { return nil } - return &internal.FrameSchema{ - Fields: encodeFields(schema.Fields), - } -} - -func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema { - if schema == nil { - return nil - } - return &FrameSchema{ - Fields: decodeFields(schema.Fields), + return &FrameOptions{ + InverseEnabled: options.InverseEnabled, + CacheType: options.CacheType, + CacheSize: options.CacheSize, + TimeQuantum: TimeQuantum(options.TimeQuantum), + Fields: decodeFields(options.Fields), } } diff --git a/frame_test.go b/frame_test.go index 6e4ade84a..78014bbce 100644 --- a/frame_test.go +++ b/frame_test.go @@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, @@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, @@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, @@ -267,49 +262,6 @@ func TestFrame_NameValidation(t *testing.T) { } } -// Ensure that frame RowLable validation is consistent. -func TestFrame_RowLabelValidation(t *testing.T) { - validRowLabels := []string{ - "", - "foo", - "hyphen-ated", - "under_score", - "abc123", - "trailing_", - "camelCase", - "UPPERCASE", - } - invalidRowLabels := []string{ - "123abc", - "x.y", - "_foo", - "-bar", - "abc def", - "a12345678901234567890123456789012345678901234567890123456789012345", - } - - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - f, err := pilosa.NewFrame(path, "i", "f") - if err != nil { - t.Fatalf("unexpected frame error: %s", err) - } - - for _, label := range validRowLabels { - if err := f.SetRowLabel(label); err != nil { - t.Fatalf("unexpected row label: %s %s", label, err) - } - } - for _, label := range invalidRowLabels { - if err := f.SetRowLabel(label); err == nil { - t.Fatalf("expected error on row label: %s", label) - } - } - -} - // Ensure frame can open and retrieve a view. func TestFrame_DeleteView(t *testing.T) { f := test.MustOpenFrame() diff --git a/gc.go b/gc.go new file mode 100644 index 000000000..23dd0f0d0 --- /dev/null +++ b/gc.go @@ -0,0 +1,41 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +// Ensure nopGCNotifier implements interface. +var _ GCNotifier = &nopGCNotifier{} + +// GCNotifier represents an interface for garbage collection notificationss. +type GCNotifier interface { + Close() + AfterGC() <-chan struct{} +} + +func init() { + NopGCNotifier = &nopGCNotifier{} +} + +// NopGCNotifier represents a GCNotifier that doesn't do anything. +var NopGCNotifier GCNotifier + +type nopGCNotifier struct{} + +// Close is a no-op implementation of GCNotifier Close method. +func (n *nopGCNotifier) Close() {} + +// AfterGC is a no-op implementation of GCNotifier AfterGC method. +func (c *nopGCNotifier) AfterGC() <-chan struct{} { + return nil +} diff --git a/gcnotify/gcnotify.go b/gcnotify/gcnotify.go new file mode 100644 index 000000000..76953a378 --- /dev/null +++ b/gcnotify/gcnotify.go @@ -0,0 +1,44 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcnotify + +import ( + "github.com/CAFxX/gcnotifier" + "github.com/pilosa/pilosa" +) + +// Ensure ActiveGCNotifier implements interface. +var _ pilosa.GCNotifier = &ActiveGCNotifier{} + +type ActiveGCNotifier struct { + gcn *gcnotifier.GCNotifier +} + +// NewActiveGCNotifier creates an active GCNotifier. +func NewActiveGCNotifier() *ActiveGCNotifier { + return &ActiveGCNotifier{ + gcn: gcnotifier.New(), + } +} + +// Close implements the GCNotifier interface. +func (n *ActiveGCNotifier) Close() { + n.gcn.Close() +} + +// AfterGC implements the GCNotifier interface. +func (n *ActiveGCNotifier) AfterGC() <-chan struct{} { + return n.gcn.AfterGC() +} diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go new file mode 100644 index 000000000..3310aeae1 --- /dev/null +++ b/gopsutil/systeminfo.go @@ -0,0 +1,115 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gopsutil + +import ( + "github.com/pilosa/pilosa" + "github.com/shirou/gopsutil/host" + "github.com/shirou/gopsutil/mem" +) + +var _ pilosa.SystemInfo = NewSystemInfo() + +// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS. +type SystemInfo struct { + platform string + family string + osVersion string +} + +// Uptime returns the system uptime in seconds. +func (s *SystemInfo) Uptime() (uptime uint64, err error) { + hostInfo, err := host.Info() + if err != nil { + return 0, err + } + return hostInfo.Uptime, nil +} + +// collectPlatformInfo fetches and caches system platform information. +func (s *SystemInfo) collectPlatformInfo() error { + var err error + if s.platform == "" { + s.platform, s.family, s.osVersion, err = host.PlatformInformation() + if err != nil { + return err + } + } + return nil +} + +// Platform returns the system platform. +func (s *SystemInfo) Platform() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.platform, nil +} + +// Family returns the system family. +func (s *SystemInfo) Family() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.family, err +} + +// OSVersion returns the OS Version. +func (s *SystemInfo) OSVersion() (string, error) { + err := s.collectPlatformInfo() + if err != nil { + return "", err + } + return s.osVersion, err +} + +// MemFree returns the amount of free memory in bytes. +func (s *SystemInfo) MemFree() (uint64, error) { + memInfo, err := mem.VirtualMemory() + if err != nil { + return 0, err + } + return memInfo.Free, err +} + +// MemTotal returns the amount of total memory in bytes. +func (s *SystemInfo) MemTotal() (uint64, error) { + memInfo, err := mem.VirtualMemory() + if err != nil { + return 0, err + } + return memInfo.Total, err +} + +// MemUsed returns the amount of used memory in bytes. +func (s *SystemInfo) MemUsed() (uint64, error) { + memInfo, err := mem.VirtualMemory() + if err != nil { + return 0, err + } + return memInfo.Used, err +} + +// KernelVersion returns the kernel version as a string. +func (s *SystemInfo) KernelVersion() (string, error) { + return host.KernelVersion() +} + +// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. +func NewSystemInfo() *SystemInfo { + return &SystemInfo{} +} diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go new file mode 100644 index 000000000..0f76b62da --- /dev/null +++ b/gopsutil/systeminfo_test.go @@ -0,0 +1,77 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gopsutil_test + +import ( + "log" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gopsutil" +) + +func TestSystemInfo(t *testing.T) { + var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo() + + // Uptime()(uint64, error) + // Platform()(string, error) + // Family()(string, error) + // OSVersion()(string, error) + // KernelVersion()(string, error) + // MemFree()(uint64, error) + // MemTotal()(uint64, error) + // MemUsed()(uint64, error) + // + uptime, err := systemInfo.Uptime() + if err != nil || uptime == 0 { + t.Fatalf("Error collecting uptime (error: %v)", err) + } + + platform, err := systemInfo.Platform() + if err != nil { + t.Fatalf("Error getting platform. (platform: %v, error: %v)", platform, err) + } + + family, err := systemInfo.Family() + if err != nil { + t.Fatalf("Error getting OS family. (family: %v, error: %v)", family, err) + } + + osversion, err := systemInfo.OSVersion() + if err != nil { + t.Fatalf("Error getting OS version. (osversion: %v, error: %v)", osversion, err) + } + + kernelversion, err := systemInfo.KernelVersion() + if err != nil { + t.Fatalf("Error getting kernel version. (kernelversion: %v, error: %v)", kernelversion, err) + } + + memfree, err := systemInfo.MemFree() + if err != nil { + t.Fatalf("Error getting memfree. (memfree: %v, error: %v)", memfree, err) + } + + memused, err := systemInfo.MemUsed() + if err != nil { + t.Fatalf("Error getting memused. (memused: %v, error: %v)", memused, err) + } + + memtotal, err := systemInfo.MemTotal() + log.Println(memtotal) + if err != nil { + t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err) + } +} diff --git a/gossip/gossip.go b/gossip/gossip.go index 3afdabe31..49b66d8a2 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -16,28 +16,32 @@ package gossip import ( "fmt" - "io" + "io/ioutil" "log" - "os" + "strconv" "strings" + "sync" "time" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) -// Ensure GossipNodeSet implements interfaces. -var _ pilosa.BroadcastReceiver = &GossipNodeSet{} -var _ pilosa.Gossiper = &GossipNodeSet{} -var _ memberlist.Delegate = &GossipNodeSet{} +// Ensure GossipMemberSet implements interfaces. +var _ pilosa.BroadcastReceiver = &GossipMemberSet{} +var _ pilosa.Gossiper = &GossipMemberSet{} +var _ memberlist.Delegate = &GossipMemberSet{} -// GossipNodeSet represents a gossip implementation of NodeSet using memberlist -// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster -// GossipNodeSet also represents an implementation of memberlist.Delegate -type GossipNodeSet struct { +// GossipMemberSet represents a gossip implementation of MemberSet using memberlist. +type GossipMemberSet struct { + mu sync.RWMutex + node *pilosa.Node memberlist *memberlist.Memberlist handler pilosa.BroadcastHandler @@ -46,50 +50,66 @@ type GossipNodeSet struct { statusHandler pilosa.StatusHandler config *gossipConfig - // The writer for any logging. - LogOutput io.Writer + Logger pilosa.Logger + + logger *log.Logger + transport *Transport } -// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. -func (g *GossipNodeSet) Nodes() []*pilosa.Node { - a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) - for _, n := range g.memberlist.Members() { - a = append(a, &pilosa.Node{Scheme: "gossip", Host: n.Name}) - } - return a -} - -// Start implements the BroadcastReceiver interface and sets the BroadcastHandler -func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { +// Start implements the BroadcastReceiver interface and sets the BroadcastHandler. +func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil } -// Seed returns the gossipSeed determined by the config. -func (g *GossipNodeSet) Seed() string { - return g.config.gossipSeed +// GetBindAddr returns the gossip bind address based on config and auto bind port. +// This method is currently only used in a test scenario where a second node needs +// the auto-bind address of the first node to use as its gossip seed. +func (g *GossipMemberSet) GetBindAddr() string { + return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort) } -// Open implements the NodeSet interface to start network activity. -func (g *GossipNodeSet) Open() error { +// Open implements the MemberSet interface to start network activity. +func (g *GossipMemberSet) Open(n *pilosa.Node) error { if g.handler == nil { - return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") + return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") } - ml, err := memberlist.Create(g.config.memberlistConfig) + + g.node = n + + err := error(nil) + g.mu.Lock() + g.memberlist, err = memberlist.Create(g.config.memberlistConfig) + g.mu.Unlock() if err != nil { return errors.Wrap(err, "creating memberlist") } - g.memberlist = ml + g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return ml.NumMembers() + g.mu.RLock() + defer g.mu.RUnlock() + return g.memberlist.NumMembers() }, RetransmitMult: 3, } - // attach to gossip seed node - nodes := []*pilosa.Node{&pilosa.Node{Scheme: "gossip", Host: g.config.gossipSeed}} //TODO: support a list of seeds - err = g.joinWithRetry(pilosa.Nodes(nodes).Hosts()) + var uris = make([]*pilosa.URI, len(g.config.gossipSeeds)) + for i, addr := range g.config.gossipSeeds { + uris[i], err = pilosa.NewURIFromAddress(addr) + if err != nil { + return fmt.Errorf("new uri from address: %s", err) + } + } + + var nodes = make([]*pilosa.Node, len(uris)) + for i, uri := range uris { + nodes[i] = &pilosa.Node{URI: *uri} + } + + g.mu.RLock() + err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings()) + g.mu.RUnlock() if err != nil { return errors.Wrap(err, "joinWithRetry") } @@ -97,7 +117,7 @@ func (g *GossipNodeSet) Open() error { } // joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *GossipNodeSet) joinWithRetry(hosts []string) error { +func (g *GossipMemberSet) joinWithRetry(hosts []string) error { err := retry(60, 2*time.Second, func() error { _, err := g.memberlist.Join(hosts) return err @@ -121,128 +141,137 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) { return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } -// logger returns a logger for the GossipNodeSet. -func (g *GossipNodeSet) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - //////////////////////////////////////////////////////////////// type gossipConfig struct { - gossipSeed string + gossipSeeds []string memberlistConfig *memberlist.Config } -// newTransport returns a NetTransport based on the memberlist configuration. -// It will dynamically bind to a port if conf.BindPort is 0. -// This is useful for test cases where specifiying a port is not reasonable. -func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { - if conf.LogOutput != nil && conf.Logger != nil { - return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.") - } +// GossipMemberSetOption describes a functional option for GossipMemberSet. +type GossipMemberSetOption func(*GossipMemberSet) error - logDest := conf.LogOutput - if logDest == nil { - logDest = os.Stderr +// WithTransport is a functional option for providing a transport to NewGossipMemberSet. +func WithTransport(transport *Transport) GossipMemberSetOption { + return func(g *GossipMemberSet) error { + g.transport = transport + return nil } - - logger := conf.Logger - if logger == nil { - logger = log.New(logDest, "", log.LstdFlags) - } - - nc := &memberlist.NetTransportConfig{ - BindAddrs: []string{conf.BindAddr}, - BindPort: conf.BindPort, - Logger: logger, - } - - // See comment below for details about the retry in here. - makeNetRetry := func(limit int) (*memberlist.NetTransport, error) { - var err error - for try := 0; try < limit; try++ { - var nt *memberlist.NetTransport - if nt, err = memberlist.NewNetTransport(nc); err == nil { - return nt, nil - } - if strings.Contains(err.Error(), "address already in use") { - logger.Printf("[DEBUG] Got bind error: %v", err) - continue - } - } - - return nil, fmt.Errorf("failed to obtain an address: %v", err) - } - - // The dynamic bind port operation is inherently racy because - // even though we are using the kernel to find a port for us, we - // are attempting to bind multiple protocols (and potentially - // multiple addresses) with the same port number. We build in a - // few retries here since this often gets transient errors in - // busy unit tests. - limit := 1 - if conf.BindPort == 0 { - limit = 10 - } - - nt, err := makeNetRetry(limit) - if err != nil { - return nil, fmt.Errorf("Could not set up network transport: %v", err) - } - if conf.BindPort == 0 { - port := nt.GetAutoBindPort() - conf.BindPort = port - conf.AdvertisePort = port - logger.Printf("[DEBUG] Using dynamic bind port %d", port) - } - - return nt, nil } -// NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipNodeSet, error) { - g := &GossipNodeSet{ - LogOutput: server.LogOutput, +// WithLogger is a functional option for providing a logger to NewGossipMemberSet. +func WithLogger(logger *log.Logger) GossipMemberSetOption { + return func(g *GossipMemberSet) error { + g.logger = logger + return nil + } +} + +// NewGossipMemberSet returns a new instance of GossipMemberSet based on options. +func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { + g := &GossipMemberSet{ + Logger: pilosa.NopLogger, } - conf := memberlist.DefaultWANConfig() - conf.BindPort = gossipPort - conf.AdvertisePort = gossipPort + // options + for _, opt := range options { + if err := opt(g); err != nil { + return nil, err + } + } + + if g.transport == nil { + port, err := strconv.Atoi(cfg.Port) + if err != nil { + return nil, fmt.Errorf("convert port: %s", err) + } + + // Set up the transport. + transport, err := NewTransport(host, port, g.logger) + if err != nil { + return nil, fmt.Errorf("new tranport: %s", err) + } + + g.transport = transport + } + + port := g.transport.Net.GetAutoBindPort() + + var gossipKey []byte + var err error + if cfg.Key != "" { + gossipKey, err = ioutil.ReadFile(cfg.Key) + if err != nil { + return nil, fmt.Errorf("reading gossip key: %s", err) + } + } + + // memberlist config + conf := memberlist.DefaultWANConfig() + conf.Transport = g.transport.Net + conf.Name = name + conf.BindAddr = host + conf.BindPort = port + conf.AdvertisePort = port + conf.AdvertiseAddr = pilosa.HostToIP(host) + // + conf.TCPTimeout = time.Duration(cfg.StreamTimeout) + conf.SuspicionMult = cfg.SuspicionMult + conf.PushPullInterval = time.Duration(cfg.PushPullInterval) + conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout) + conf.ProbeInterval = time.Duration(cfg.ProbeInterval) + conf.GossipNodes = cfg.Nodes + conf.GossipInterval = time.Duration(cfg.Interval) + conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime) + // + conf.Delegate = g + conf.SecretKey = gossipKey + conf.Events = ger + conf.Logger = g.logger - //TODO: pull memberlist config from pilosa.cfg file g.config = &gossipConfig{ memberlistConfig: conf, - gossipSeed: gossipSeed, + gossipSeeds: cfg.Seeds, } - g.config.memberlistConfig.Name = name - g.config.memberlistConfig.BindAddr = gossipHost - g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost) - g.config.memberlistConfig.Delegate = g - g.config.memberlistConfig.SecretKey = secretKey - - g.statusHandler = server - - // set up the transport - transport, err := newTransport(g.config.memberlistConfig) - if err != nil { - return nil, err - } - g.config.memberlistConfig.Transport = transport - - // If no gossipSeed is provided, use local host:port. - if gossipSeed == "" { - g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, g.config.memberlistConfig.BindPort) - } + g.statusHandler = sh return g, nil } -// SendAsync implementation of the Gossiper interface. -func (g *GossipNodeSet) SendAsync(pb proto.Message) error { +// SendSync implementation of the Broadcaster interface. +func (g *GossipMemberSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { - return err + return fmt.Errorf("marshal message: %s", err) + } + + mlist := g.memberlist + + // Direct sends the message directly to every node. + // An error from any node raises an error on the entire operation. + // + // Gossip uses the gossip protocol to eventually deliver the message + // to every node. + var eg errgroup.Group + for _, n := range mlist.Members() { + // Don't send the message to the local node. + if n == mlist.LocalNode() { + continue + } + node := n + eg.Go(func() error { + return mlist.SendToTCP(node, msg) + }) + } + return eg.Wait() +} + +// SendAsync implementation of the Gossiper interface. +func (g *GossipMemberSet) SendAsync(pb proto.Message) error { + msg, err := pilosa.MarshalMessage(pb) + if err != nil { + return fmt.Errorf("marshal message: %s", err) } b := &broadcast{ @@ -254,60 +283,136 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { } // NodeMeta implementation of the memberlist.Delegate interface. -func (g *GossipNodeSet) NodeMeta(limit int) []byte { - return []byte{} +func (g *GossipMemberSet) NodeMeta(limit int) []byte { + buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) + if err != nil { + g.Logger.Printf("marshal message error: %s", err) + return []byte{} + } + return buf } // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. -func (g *GossipNodeSet) NotifyMsg(b []byte) { +func (g *GossipMemberSet) NotifyMsg(b []byte) { m, err := pilosa.UnmarshalMessage(b) if err != nil { - g.logger().Printf("unmarshal message error: %s", err) + g.Logger.Printf("unmarshal message error: %s", err) return } if err := g.handler.ReceiveMessage(m); err != nil { - g.logger().Printf("receive message error: %s", err) + g.Logger.Printf("receive message error: %s", err) return } } // GetBroadcasts implementation of the memberlist.Delegate interface // called when user data messages can be broadcast. -func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. -func (g *GossipNodeSet) LocalState(join bool) []byte { +func (g *GossipMemberSet) LocalState(join bool) []byte { pb, err := g.statusHandler.LocalStatus() if err != nil { - g.logger().Printf("error getting local state, err=%s", err) + g.Logger.Printf("error getting local state, err=%s", err) return []byte{} } // Marshal nodestate data to bytes. buf, err := proto.Marshal(pb) if err != nil { - g.logger().Printf("error marshalling nodestate data, err=%s", err) + g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} } return buf } // MergeRemoteState implementation of the memberlist.Delegate interface -// receive and process the remote side side's LocalState. -func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { +// receive and process the remote side's LocalState. +func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus if err := proto.Unmarshal(buf, &pb); err != nil { - g.logger().Printf("error unmarshalling nodestate data, err=%s", err) + g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) return } err := g.statusHandler.HandleRemoteStatus(&pb) if err != nil { - g.logger().Printf("merge state error: %s", err) + g.Logger.Printf("merge state error: %s", err) + } +} + +// GossipEventReceiver is used to enable an application to receive +// events about joins and leaves over a channel. +// +// Care must be taken that events are processed in a timely manner from +// the channel, since this delegate will block until an event can be sent. +type GossipEventReceiver struct { + ch chan memberlist.NodeEvent + eventHandler pilosa.EventHandler + + Logger pilosa.Logger +} + +// NewGossipEventReceiver returns a new instance of GossipEventReceiver. +func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver { + return &GossipEventReceiver{ + ch: make(chan memberlist.NodeEvent, 1), + Logger: logger, + } +} + +func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n} +} + +func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n} +} + +func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) { + g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n} +} + +// Start implements the pilosa.EventReceiver interface and sets the EventHandler. +func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { + g.eventHandler = h + go g.listen() + return nil +} + +func (g *GossipEventReceiver) listen() { + var nodeEventType pilosa.NodeEventType + for { + e := <-g.ch + switch e.Event { + case memberlist.NodeJoin: + nodeEventType = pilosa.NodeJoin + case memberlist.NodeLeave: + nodeEventType = pilosa.NodeLeave + case memberlist.NodeUpdate: + nodeEventType = pilosa.NodeUpdate + default: + continue + } + + // Get the node from the event.Node meta data. + var n internal.Node + if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { + panic("failed to unmarshal event node meta data") + } + node := pilosa.DecodeNode(&n) + + ne := &pilosa.NodeEvent{ + Event: nodeEventType, + Node: node, + } + if err := g.eventHandler.ReceiveEvent(ne); err != nil { + g.Logger.Printf("receive event error: %s", err) + } } } @@ -330,3 +435,148 @@ func (b *broadcast) Finished() { close(b.notify) } } + +// Transport is a gossip transport for binding to a port. +type Transport struct { + //memberlist.Transport + Net *memberlist.NetTransport + URI *pilosa.URI +} + +// NewTransport returns a NetTransport based on the given host and port. +// It will dynamically bind to a port if port is 0. +// This is useful for test cases where specifying a port is not reasonable. +//func NewTransport(host string, port int) (*memberlist.NetTransport, error) { +func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { + // memberlist config + conf := memberlist.DefaultWANConfig() + conf.BindAddr = host + conf.BindPort = port + conf.AdvertisePort = port + conf.Logger = logger + + net, err := newTransport(conf) + if err != nil { + return nil, fmt.Errorf("new transport: %s", err) + } + + uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort())) + if err != nil { + return nil, fmt.Errorf("new uri from host port: %s", err) + } + + return &Transport{ + Net: net, + URI: uri, + }, nil +} + +// newTransport returns a NetTransport based on the memberlist configuration. +// It will dynamically bind to a port if conf.BindPort is 0. +func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { + nc := &memberlist.NetTransportConfig{ + BindAddrs: []string{conf.BindAddr}, + BindPort: conf.BindPort, + Logger: conf.Logger, + } + + // See comment below for details about the retry in here. + makeNetRetry := func(limit int) (*memberlist.NetTransport, error) { + var err error + for try := 0; try < limit; try++ { + var nt *memberlist.NetTransport + if nt, err = memberlist.NewNetTransport(nc); err == nil { + return nt, nil + } + if strings.Contains(err.Error(), "address already in use") { + conf.Logger.Printf("[DEBUG] Got bind error: %v", err) + continue + } + } + + return nil, fmt.Errorf("failed to obtain an address: %v", err) + } + + // The dynamic bind port operation is inherently racy because + // even though we are using the kernel to find a port for us, we + // are attempting to bind multiple protocols (and potentially + // multiple addresses) with the same port number. We build in a + // few retries here since this often gets transient errors in + // busy unit tests. + limit := 1 + if conf.BindPort == 0 { + limit = 10 + } + + nt, err := makeNetRetry(limit) + if err != nil { + return nil, fmt.Errorf("Could not set up network transport: %v", err) + } + + return nt, nil +} + +// Config holds toml-friendly memberlist configuration. +type Config struct { + // Port indicates the port to which pilosa should bind for internal state sharing. + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + StreamTimeout toml.Duration `toml:"stream-timeout"` + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + SuspicionMult int `toml:"suspicion-mult"` + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. + PushPullInterval toml.Duration `toml:"push-pull-interval"` + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + ProbeInterval toml.Duration `toml:"probe-interval"` + ProbeTimeout toml.Duration `toml:"probe-timeout"` + + // Interval and Nodes are used to configure the gossip + // behavior of memberlist. + // + // Interval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // ToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + Interval toml.Duration `toml:"interval"` + Nodes int `toml:"nodes"` + ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` +} diff --git a/handler.go b/handler.go index e5cce4648..871ba885a 100644 --- a/handler.go +++ b/handler.go @@ -12,64 +12,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate statik -src=./webui - package pilosa import ( - "context" - "encoding/csv" "encoding/json" - "errors" "expvar" "fmt" "io" "io/ioutil" - "log" "net/http" + "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" - "os" + "reflect" "runtime/debug" "strconv" "strings" "time" - - "reflect" + "unicode" "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - - "unicode" - - // Allow building Pilosa without the web UI. - _ "github.com/pilosa/pilosa/statik" - "github.com/rakyll/statik/fs" + "github.com/pkg/errors" ) // Handler represents an HTTP handler. type Handler struct { - Holder *Holder - Broadcaster Broadcaster - BroadcastHandler BroadcastHandler - StatusHandler StatusHandler - - // Local hostname & cluster configuration. - URI *URI - Cluster *Cluster - RemoteClient *http.Client - Router *mux.Router - // The execution engine for running queries. - Executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) - } + FileSystem FileSystem - // The writer for any logging. - LogOutput io.Writer + Logger Logger + + // Keeps the query argument validators for each handler + validators map[string]*queryValidationSpec + + API *API } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -85,28 +64,81 @@ var externalPrefixFlag = map[string]bool{ "version": true, } +type errorResponse struct { + Error string `json:"error"` +} + // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ - LogOutput: os.Stderr, + FileSystem: NopFileSystem, + Logger: NopLogger, } handler.Router = NewRouter(handler) + handler.populateValidators() return handler } -// NewRouter creates a Gorilla Mux http router. +func (h *Handler) populateValidators() { + h.validators = map[string]*queryValidationSpec{} + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") + h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFrameRestore"] = queryValidationSpecRequired("host") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { + if err := validator.validate(r.URL.Query()); err != nil { + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) +} + +// NewRouter creates a new mux http router. func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") + router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") + router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") + router.HandleFunc("/id", handler.handleGetID).Methods("GET") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET") - router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET") - router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET") + router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData") + router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") @@ -118,8 +150,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") + router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") @@ -129,16 +160,8 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") - router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") - router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/id", handler.handleGetID).Methods("GET") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: @@ -146,6 +169,7 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.Use(handler.queryArgValidator) return router } @@ -160,7 +184,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) stack := debug.Stack() msg := "PANIC: %s\n%s" - fmt.Fprintf(h.LogOutput, msg, err, stack) + h.Logger.Printf(msg, err, stack) fmt.Fprintf(w, msg, err, stack) } }() @@ -170,25 +194,25 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { dif := time.Since(t) // Calculate per request StatsD metrics when the handler is fully configured. - if h.Holder != nil && h.Cluster != nil { - statsTags := make([]string, 0, 3) + statsTags := make([]string, 0, 3) - if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { - h.logger().Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } + longQueryTime := h.API.LongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") + } - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - - stats := h.Holder.Stats.WithTags(statsTags...) + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + stats := h.API.StatsWithTags(statsTags) + if stats != nil { stats.Histogram("http."+endpointName, float64(dif), 0.1) } } @@ -199,35 +223,40 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) return } - statikFS, err := fs.New() + filesystem, err := h.FileSystem.New() if err != nil { - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } - http.FileServer(statikFS).ServeHTTP(w, r) + http.FileServer(filesystem).ServeHTTP(w, r) } // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: h.Holder.Schema(), + Indexes: schema, }); err != nil { - h.logger().Printf("write schema response error: %s", err) + h.Logger.Printf("write schema response error: %s", err) } } // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status, err := h.StatusHandler.ClusterStatus() - if err != nil { - h.logger().Printf("cluster status error: %s", err) - return + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), } - if err := json.NewEncoder(w).Encode(getStatusResponse{ - Status: status, - }); err != nil { - h.logger().Printf("write status response error: %s", err) + if err := json.NewEncoder(w).Encode(status); err != nil { + h.Logger.Printf("write status response error: %s", err) + } +} + +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + info := h.API.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.Logger.Printf("write info response error: %s", err) } } @@ -236,13 +265,12 @@ type getSchemaResponse struct { } type getStatusResponse struct { - Status proto.Message `json:"status"` + State string `json:"state"` + Nodes []*Node `json:"nodes"` } // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - // Parse incoming request. req, err := h.readQueryRequest(r) if err != nil { @@ -250,48 +278,16 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } + // TODO: Remove + req.Index = mux.Vars(r)["index"] - // Build execution options. - opt := &ExecOptions{ - Remote: req.Remote, - ExcludeAttrs: req.ExcludeAttrs, - ExcludeBits: req.ExcludeBits, - } - - // Parse query string. - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + resp, err := h.API.Query(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } - // Execute the query. - results, err := h.Executor.Execute(r.Context(), indexName, q, req.Slices, opt) - resp := &QueryResponse{Results: results, Err: err} - - // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeBits { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Bitmap) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := h.readColumnAttrSets(h.Holder.Index(indexName), columnIDs) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - resp.ColumnAttrSets = columnAttrSets - } - // Set appropriate status code, if there is an error. if resp.Err != nil { switch resp.Err { @@ -303,36 +299,24 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Write response back to client. - if err := h.writeQueryResponse(w, r, resp); err != nil { - h.logger().Printf("write query response error: %s", err) + if err := h.writeQueryResponse(w, r, &resp); err != nil { + h.Logger.Printf("write query response error: %s", err) } } -func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - var ms map[string]uint64 - if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { - ms = h.Holder.MaxInverseSlices() - } else { - ms = h.Holder.MaxSlices() +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + Inverse: h.API.MaxInverseSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) } - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - pb := &internal.MaxSlicesResponse{ - MaxSlices: ms, - } - if buf, err := proto.Marshal(pb); err != nil { - h.logger().Printf("protobuf marshal error: %s", err) - } else if _, err := w.Write(buf); err != nil { - h.logger().Printf("stream write error: %s", err) - } - return - } - json.NewEncoder(w).Encode(sliceMaxResponse{ - MaxSlices: ms, - }) } -type sliceMaxResponse struct { - MaxSlices map[string]uint64 `json:"maxSlices"` +type getSlicesMaxResponse struct { + Standard map[string]uint64 `json:"standard"` + Inverse map[string]uint64 `json:"inverse"` } // handleGetIndexes handles GET /index request. @@ -343,16 +327,16 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + index, err := h.API.Index(r.Context(), indexName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } if err := json.NewEncoder(w).Encode(getIndexResponse{ map[string]string{"name": index.Name()}, }); err != nil { - h.logger().Printf("write response error: %s", err) + h.Logger.Printf("write response error: %s", err) } } @@ -427,28 +411,17 @@ type postIndexResponse struct{} // handleDeleteIndex handles DELETE /index request. func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - - // Delete index from the holder. - if err := h.Holder.DeleteIndex(indexName); err != nil { + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete index message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteIndexMessage{ - Index: indexName, - }) - if err != nil { - h.logger().Printf("problem sending DeleteIndex message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.Count("deleteIndex", 1, 1.0) } type deleteIndexResponse struct{} @@ -468,8 +441,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Create index. - _, err = h.Holder.CreateIndex(indexName, req.Options) + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) if err == ErrIndexExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -478,69 +450,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Send the create index message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateIndexMessage{ - Index: indexName, - Meta: req.Options.Encode(), - }) - if err != nil { - h.logger().Printf("problem sending CreateIndex message: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // Encode response. if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } - - h.Holder.Stats.Count("createIndex", 1, 1.0) -} - -// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. -func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req patchIndexTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Retrieve index by name. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := index.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } -type patchIndexTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchIndexTimeQuantumResponse struct{} - // handlePostIndexAttrDiff handles POST /index/attr/diff requests. func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] @@ -552,41 +467,21 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := index.ColumnAttrStore().Blocks() + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := index.ColumnAttrStore().BlockData(blockID) - if err != nil { + if err == ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -613,42 +508,22 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Create frame. - _, err = index.CreateFrame(frameName, req.Options) - if err == ErrFrameExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Send the create frame message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateFrameMessage{ - Index: indexName, - Frame: frameName, - Meta: req.Options.Encode(), - }) + _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) if err != nil { - h.logger().Printf("problem sending CreateFrame message: %s", err) + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrFrameExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return } - // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) - } type _postFrameRequest postFrameRequest @@ -701,85 +576,26 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + err := h.API.DeleteFrame(r.Context(), indexName, frameName) + if err != nil { + if err == ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return } - return - } - - // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete frame message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ - Index: indexName, - Frame: frameName, - }) - if err != nil { - h.logger().Printf("problem sending DeleteFrame message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) } type deleteFrameResponse struct{} -// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. -func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req patchFrameTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Retrieve index by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := f.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } -} - -type patchFrameTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchFrameTimeQuantumResponse struct{} - // handlePostFrameField handles POST /frame/field request. func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] @@ -793,27 +609,25 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { return } - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Create new field. - if err := f.CreateField(&Field{ + field := &Field{ Name: fieldName, Type: req.Type, Min: req.Min, Max: req.Max, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + } + + if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -831,22 +645,18 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) frameName := mux.Vars(r)["frame"] fieldName := mux.Vars(r)["field"] - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Delete field. - if err := f.DeleteField(fieldName); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -854,30 +664,22 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - index := h.Holder.index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - frame := index.frame(frameName) - if frame == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - schema, err := frame.GetFields() - if err == ErrFrameFieldsNotAllowed { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + fields, err := h.API.Fields(r.Context(), indexName, frameName) + if err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } // Encode response. - if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: schema.Fields}); err != nil { - h.logger().Printf("response encoding error: %s", err) + if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { + h.Logger.Printf("response encoding error: %s", err) } } @@ -894,15 +696,16 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Retrieve views. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + views, err := h.API.Views(r.Context(), indexName, frameName) + if err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Fetch views. - views := f.Views() names := make([]string, len(views)) for i := range views { names[i] = views[i].Name() @@ -910,7 +713,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { // Encode response. if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -920,36 +723,18 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { frameName := mux.Vars(r)["frame"] viewName := mux.Vars(r)["view"] - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Delete the view. - if err := f.DeleteView(viewName); err != nil { - // Ingore this error becuase views do not exist on all nodes due to slice distribution. - if err != ErrInvalidView { + if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusBadRequest) - return } - } - - // Send the delete view message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteViewMessage{ - Index: indexName, - Frame: frameName, - View: viewName, - }) - if err != nil { - h.logger().Printf("problem sending DeleteView message: %s", err) + return } // Encode response. if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -971,41 +756,22 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := f.RowAttrStore().Blocks() + attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := f.RowAttrStore().BlockData(blockID) - if err != nil { + switch err { + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ Attrs: attrs, }); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1070,12 +836,6 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro // readURLQueryRequest parses query parameters from URL parameters from r. func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() - validQuery := validOptions(QueryRequest{}) - for key := range q { - if _, ok := validQuery[key]; !ok { - return nil, errors.New("invalid query params") - } - } // Parse query string. buf, err := ioutil.ReadAll(r.Body) @@ -1162,44 +922,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Convert timestamps to time.Time. - timestamps := make([]*time.Time, len(req.Timestamps)) - for i, ts := range req.Timestamps { - if ts == 0 { - continue + if err := h.API.Import(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } - t := time.Unix(0, ts) - timestamps[i] = &t - } - - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) - http.Error(w, mesg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) - if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -1242,34 +975,17 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice) - http.Error(w, mesg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.logger().Println("importing:", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.ImportValue(req.Field, req.ColumnIDs, req.Values) - if err != nil { - h.logger().Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1308,35 +1024,17 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), index, slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice) - http.Error(w, mesg, http.StatusPreconditionFailed) + if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { + switch err { + case ErrFragmentNotFound: + break + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - - // Find the fragment. - f := h.Holder.Fragment(index, frame, view, slice) - if f == nil { - return - } - - // Wrap writer with a CSV writer. - cw := csv.NewWriter(w) - - // Iterate over each bit. - if err := f.ForEachBit(func(rowID, columnID uint64) error { - return cw.Write([]string{ - strconv.FormatUint(rowID, 10), - strconv.FormatUint(columnID, 10), - }) - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Ensure data is flushed. - cw.Flush() } // handleGetFragmentNodes handles /fragment/nodes requests. @@ -1347,20 +1045,24 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) // Read slice parameter. slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) + http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) return } // Retrieve fragment owner nodes. - nodes := h.Cluster.FragmentNodes(index, slice) + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.logger().Printf("json write error: %s", err) + h.Logger.Printf("json write error: %s", err) } } -// handleGetFragmentBackup handles GET /fragment/data requests. +// handleGetFragmentData handles GET /fragment/data requests. func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { // Read slice parameter. q := r.URL.Query() @@ -1371,19 +1073,19 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } // Stream fragment to response body. if _, err := f.WriteTo(w); err != nil { - h.logger().Printf("fragment backup error: %s", err) + h.Logger.Printf("fragment backup error: %s", err) } } -// handlePostFragmentRestore handles POST /fragment/data requests. +// handlePostFragmentData handles POST /fragment/data requests. func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) { // Read slice parameter. q := r.URL.Query() @@ -1393,63 +1095,26 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - // Retrieve frame. - f := h.Holder.Frame(q.Get("index"), q.Get("frame")) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve view. - view, err := f.CreateViewIfNotExists(q.Get("view")) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve fragment from frame. - frag, err := view.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read fragment in from request body. - if _, err := frag.ReadFrom(r.Body); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { + if err == ErrFrameNotFound { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } } -// handleGetFragmentData handles GET /fragment/block/data requests. +// handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - // Read request object. - var req internal.BlockDataRequest - if body, err := ioutil.ReadAll(r.Body); err != nil { - http.Error(w, "ready body error", http.StatusBadRequest) - return - } else if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, "unmarshal body error", http.StatusBadRequest) - return - } - - // Retrieve fragment from holder. - f := h.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) - if f == nil { - http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) - return - } - - // Read data - var resp internal.BlockDataResponse - if f != nil { - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) - } - - // Encode response. - buf, err := proto.Marshal(&resp) + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) if err != nil { - h.logger().Printf("merge block response encoding error: %s", err) + if _, ok := err.(BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1469,21 +1134,21 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Retrieve blocks. - blocks := f.Blocks() - // Encode response. if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, }); err != nil { - h.logger().Printf("block response encoding error: %s", err) + h.Logger.Printf("block response encoding error: %s", err) } } @@ -1510,119 +1175,37 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusBadRequest) } - // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, h.RemoteClient) - - // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByIndex(r.Context()) - if err != nil { - http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve list of all views. - views, err := client.FrameViews(r.Context(), indexName, frameName) - if err != nil { - http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) - return - } - - // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { - // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.URI.HostPort(), indexName, slice) { - continue - } - - // Loop over view names. - for _, view := range views { - // Create view. - v, err := f.CreateViewIfNotExists(view) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Otherwise retrieve the local fragment. - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), indexName, frameName, view, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return err - } - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } + err = h.API.RestoreFrame(r.Context(), indexName, frameName, host) + switch err { + case nil: + break + case ErrFrameNotFound: + fallthrough + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } } // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { - h.logger().Printf("write version response error: %s", err) + hosts := h.API.Hosts(r.Context()) + if err := json.NewEncoder(w).Encode(hosts); err != nil { + h.Logger.Printf("write version response error: %s", err) } } // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - version := Version - if strings.HasPrefix(version, "v") { - // make the version string semver-compatible - version = version[1:] - } - if err := json.NewEncoder(w).Encode(struct { + err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: version, - }); err != nil { - h.logger().Printf("write version response error: %s", err) - } -} - -// handleExpvar handles /debug/vars requests. -func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) { - // Copied from $GOROOT/src/expvar/expvar.go - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprintf(w, "{\n") - first := true - expvar.Do(func(kv expvar.KeyValue) { - if !first { - fmt.Fprintf(w, ",\n") - } - first = false - fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) + Version: h.API.Version(), }) - fmt.Fprintf(w, "\n}\n") -} - -// logger returns a logger for the handler. -func (h *Handler) logger() *log.Logger { - return log.New(h.LogOutput, "", log.LstdFlags) + if err != nil { + h.Logger.Printf("write version response error: %s", err) + } } // QueryResult types. @@ -1630,7 +1213,7 @@ const ( QueryResultTypeNil uint32 = iota QueryResultTypeBitmap QueryResultTypePairs - QueryResultTypeSumCount + QueryResultTypeValCount QueryResultTypeUint64 QueryResultTypeBool ) @@ -1719,9 +1302,9 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Type = QueryResultTypePairs pb.Results[i].Pairs = encodePairs(result) - case SumCount: - pb.Results[i].Type = QueryResultTypeSumCount - pb.Results[i].SumCount = encodeSumCount(result) + case ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = encodeValCount(result) case uint64: pb.Results[i].Type = QueryResultTypeUint64 pb.Results[i].N = result @@ -1772,13 +1355,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1787,36 +1363,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - if err := req.Validate(); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + if err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req); err != nil { + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: + http.Error(w, err.Error(), http.StatusConflict) + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Encode InputDefinition to its internal representation. - def := req.Encode() - def.Name = inputDefName - - // Create InputDefinition. - _, err = index.CreateInputDefinition(def) - if err == ErrInputDefinitionExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = h.Broadcaster.SendSync( - &internal.CreateInputDefinitionMessage{ - Index: indexName, - Definition: def, - }) - if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) - } - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1825,16 +1395,18 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - inputDef, err := index.InputDefinition(inputDefName) + inputDef, err := h.API.InputDefinition(r.Context(), indexName, inputDefName) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1842,9 +1414,8 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques Frames: inputDef.frames, Fields: inputDef.fields, }); err != nil { - h.logger().Printf("write status response error: %s", err) + h.Logger.Printf("write status response error: %s", err) } - } // handleDeleteInputDefinition handles DELETE /input-definition request. @@ -1852,30 +1423,22 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + if err := h.API.DeleteInputDefinition(r.Context(), indexName, inputDefName); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusNotFound) + } return } - // Delete input definition from the index. - if err := index.DeleteInputDefinition(inputDefName); err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - err := h.Broadcaster.SendSync( - &internal.DeleteInputDefinitionMessage{ - Index: indexName, - Name: inputDefName, - }) - if err != nil { - h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) - } - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1885,13 +1448,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1899,99 +1455,133 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - for _, req := range reqs { - bits, err := h.InputJSONDataParser(req.(map[string]interface{}), index, inputDefName) - if err == ErrInputDefinitionNotFound { + + if err = h.API.WriteInput(r.Context(), indexName, inputDefName, reqs); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: http.Error(w, err.Error(), http.StatusNotFound) - return - } else if err != nil { + default: http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for fr, bs := range bits { - err := index.InputBits(fr, bs) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } } + return } + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } -// InputJSONDataParser validates input json file and executes SetBit. -func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { - inputDef, err := index.InputDefinition(name) +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) if err != nil { - return nil, err - } - // If field in input data is not in defined definition, return error. - var colValue uint64 - validFields := make(map[string]bool) - timestampFrame := make(map[string]int64) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - if field.PrimaryKey { - value, ok := req[field.Name] - if !ok { - return nil, fmt.Errorf("primary key does not exist") - } - rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } - colValue = uint64(rawValue) - } - // Find frame that need to add timestamp. - for _, action := range field.Actions { - if action.ValueDestination == InputSetTimestamp { - timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) - if err != nil { - return nil, err - } - } - } + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) + return } - for key := range req { - _, ok := validFields[key] - if !ok { - return nil, fmt.Errorf("field not found: %s", key) + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, + New: newNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + ID string `json:"id"` +} + +type setCoordinatorResponse struct { + Old *Node `json:"old"` + New *Node `json:"new"` +} + +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } - setBits := make(map[string][]*Bit) - - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue + removeNode, err := h.API.RemoveNode(req.ID) + if err != nil { + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) } + return + } - // Looking into timestampFrame map and set timestamp to the whole frame - for _, action := range field.Actions { - frame := action.Frame - timestamp := timestampFrame[action.Frame] - // Skip input data field values that are set to null - if req[field.Name] == nil { - continue - } - bit, err := HandleAction(action, req[field.Name], colValue, timestamp) - if err != nil { - return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) - } - if bit != nil { - setBits[frame] = append(setBits[frame], bit) - } + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + ID string `json:"id"` +} + +type removeNodeResponse struct { + Remove *Node `json:"remove"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() + var msg string + if err != nil { + switch errors.Cause(err) { + case ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return } } - return setBits, nil + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` } func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - h.Holder.RecalculateCaches() + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) } @@ -2018,42 +1608,64 @@ func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { - fmt.Println("**unsupported media type**") http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return } - // Read entire body. - body, err := ioutil.ReadAll(r.Body) + err := h.API.ClusterMessage(r.Context(), r.Body) if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - pb, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Forward the error message. - err = h.BroadcastHandler.ReceiveMessage(pb) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.Holder.LocalID)) + _, err := w.Write([]byte(h.API.LocalID())) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } } type defaultClusterMessageResponse struct{} + +type queryValidationSpec struct { + required []string + args map[string]struct{} +} + +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]struct{}{} + for _, arg := range requiredArgs { + args[arg] = struct{}{} + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, + } +} + +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { + for _, arg := range args { + s.args[arg] = struct{}{} + } + return s +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.Errorf("%s is required", req) + } + } + for k := range query { + if _, ok := s.args[k]; !ok { + return errors.Errorf("%s is not a valid argument", k) + } + } + return nil +} diff --git a/handler_internal_test.go b/handler_internal_test.go index 169267c38..4603662e6 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -30,8 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}}, - {json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, } for _, test := range tests { actual := &postIndexRequest{} @@ -66,11 +65,10 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, - {json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, - {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, + {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, + {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { actual := &postFrameRequest{} diff --git a/handler_test.go b/handler_test.go index 04903c979..963c0cccc 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -31,18 +32,19 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/test" ) func TestHandlerPanics(t *testing.T) { h := test.NewHandler() - buf := &bytes.Buffer{} - h.Handler.LogOutput = buf + bufLogger := test.NewBufferLogger() + h.Handler.Logger = bufLogger w := httptest.NewRecorder() // will panic since Handler has no Holder set up h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) - bufbytes, err := ioutil.ReadAll(buf) + bufbytes, err := bufLogger.ReadAll() if err != nil { t.Fatalf("reading all logoutput: %v", err) } @@ -64,8 +66,8 @@ func TestHandler_NotFound(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -99,13 +101,14 @@ func TestHandler_Schema(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -137,20 +140,55 @@ func TestHandler_Status(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - h.StatusHandler = s + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateNormal) + h.API.StatusHandler = s s.Handler = h w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } +func TestHandler_Info(t *testing.T) { + s := test.NewServer() + defer s.Close() + h := test.NewHandler() + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { + t.Fatalf("unexpected body: %s", body) + } +} + +// Ensure the handler can abort a cluster resize. +func TestHandler_ClusterResizeAbort(t *testing.T) { + + t.Run("No resize job", func(t *testing.T) { + h := test.NewHandler() + h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateResizing) + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + if w.Code != http.StatusOK { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + +} + // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() @@ -165,13 +203,13 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -206,13 +244,13 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -223,8 +261,8 @@ func TestHandler_Query_Args_URL(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -251,8 +289,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -291,8 +329,8 @@ func TestHandler_Query_Args_Err(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { @@ -306,7 +344,7 @@ func TestHandler_Query_Params_Err(t *testing.T) { test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) } @@ -318,8 +356,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -339,8 +377,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -357,7 +395,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } @@ -369,8 +407,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -402,8 +440,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -425,8 +463,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -445,7 +483,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { @@ -473,8 +511,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -506,7 +544,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -534,8 +572,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -558,8 +596,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -579,7 +617,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } @@ -591,15 +629,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -612,8 +650,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -622,7 +660,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -640,8 +678,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { @@ -655,8 +693,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -672,7 +710,7 @@ func TestHandler_Index_Delete(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -712,8 +750,8 @@ func TestHandler_DeleteFrame(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -725,57 +763,13 @@ func TestHandler_DeleteFrame(t *testing.T) { } } -// Ensure handler can set the Index time quantum. -func TestHandler_SetIndexTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - - h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - -// Ensure handler can set the frame time quantum. -func TestHandler_SetFrameTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - // Ensure the handler can return data in differing blocks for an index. func TestHandler_Index_AttrStore_Diff(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -824,7 +818,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -874,12 +868,12 @@ func TestHandler_Frame_AddField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -904,7 +898,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldType", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -926,7 +920,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldRange", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -949,8 +943,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrFieldAlreadyExists", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, + Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, }); err != nil { t.Fatal(err) } @@ -978,12 +971,12 @@ func TestHandler_Frame_DeleteField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1011,7 +1004,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { t.Run("ErrFieldNotFound", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1043,21 +1036,18 @@ func TestHandler_Frame_GetFields(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { t.Fatal(err) } resp, err := http.Get(s.URL + "/index/i/frame/f/fields") - if err != nil { - t.Fatal(err) - } if err != nil { t.Fatal(err) } else if resp.StatusCode != http.StatusOK { @@ -1085,7 +1075,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false}) + _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") if err != nil { @@ -1093,12 +1083,12 @@ func TestHandler_Frame_GetFields(t *testing.T) { } if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusBadRequest { + } else if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %d", resp.StatusCode) } else if body, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `frame fields not allowed` { - t.Fatalf("unexpected body: %q", body) + } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { + t.Fatalf("shouldn't get frame fields not allowed error: %q", body) } }) @@ -1114,7 +1104,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set bits in the index. @@ -1163,8 +1153,8 @@ func TestHandler_Version(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/version", nil) @@ -1186,17 +1176,33 @@ func TestHandler_Fragment_Nodes(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(3) - h.Cluster.ReplicaN = 2 + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(3) + h.API.Cluster.ReplicaN = 2 w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + + // invalid argument should return BadRequest + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) } } @@ -1206,8 +1212,8 @@ func TestHandler_Expvars(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) @@ -1252,8 +1258,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) { ] }`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1287,8 +1293,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) //Ensure throwing error if there's duplicated primaryKey field invalidPrimaryKey := []byte(` @@ -1322,7 +1328,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { } // Ensure throwing error if there's no primary key - hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) unmatchColumnBody := []byte(` { "frames":[{ @@ -1392,8 +1398,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Test index not found. w := httptest.NewRecorder() @@ -1406,7 +1412,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { // Test input definition is deleted. index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -1440,10 +1446,10 @@ func TestHandler_GetInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -1607,8 +1613,8 @@ func TestHandler_CreateInput(t *testing.T) { "null_value": null }]`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Return error if index does not exist. w := httptest.NewRecorder() @@ -1722,8 +1728,8 @@ func TestInput_JSON(t *testing.T) { err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) for _, req := range tests { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json)))) @@ -1783,8 +1789,8 @@ func TestHandler_DeleteView(t *testing.T) { hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD") h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -1809,8 +1815,8 @@ func TestHandler_RecalculateCaches(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) @@ -1819,3 +1825,31 @@ func TestHandler_RecalculateCaches(t *testing.T) { } } + +func TestHandler_WebUI(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + + h := test.NewHandler() + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.FileSystem = &statik.FileSystem{} + + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + if !strings.Contains(w.Body.String(), "Pilosa WebUI") { + t.Fatalf("WebUI is not being served correctly.") + } + + // If curl is the client, the response should be different + w = httptest.NewRecorder() + req := test.MustNewHTTPRequest("GET", "/", nil) + req.Header.Add("User-Agent", "curl/7.54.0") + h.ServeHTTP(w, req) + if !strings.Contains(w.Body.String(), "try the WebUI") { + t.Fatalf("WebUI is not being served correctly.") + } +} diff --git a/holder.go b/holder.go index c91c3d07d..2475d61d0 100644 --- a/holder.go +++ b/holder.go @@ -16,11 +16,8 @@ package pilosa import ( "context" - "errors" "fmt" - "io" "io/ioutil" - "log" "net/http" "os" "path" @@ -31,6 +28,8 @@ import ( "syscall" "time" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -49,7 +48,13 @@ type Holder struct { // Indexes by name. indexes map[string]*Index + // opened channel is closed once Open() completes. + opened chan struct{} + Broadcaster Broadcaster + + NewAttrStore func(string) AttrStore + // Close management wg sync.WaitGroup closing chan struct{} @@ -63,9 +68,7 @@ type Holder struct { // The interval at which the cached row ids are persisted to disk. CacheFlushInterval time.Duration - LogOutput io.Writer - - LocalID string + Logger Logger } // NewHolder returns a new instance of Holder. @@ -74,12 +77,16 @@ func NewHolder() *Holder { indexes: make(map[string]*Index), closing: make(chan struct{}, 0), + opened: make(chan struct{}), + Broadcaster: NopBroadcaster, Stats: NopStatsClient, + NewAttrStore: NewNopAttrStore, + CacheFlushInterval: DefaultCacheFlushInterval, - LogOutput: os.Stderr, + Logger: NopLogger, } } @@ -87,20 +94,21 @@ func NewHolder() *Holder { func (h *Holder) Open() error { h.setFileLimit() + h.Logger.Printf("open holder path: %s", h.Path) if err := os.MkdirAll(h.Path, 0777); err != nil { - return err + return errors.Wrap(err, "creating directory") } // Open path to read all index directories. f, err := os.Open(h.Path) if err != nil { - return err + return errors.Wrap(err, "opening directory") } defer f.Close() fis, err := f.Readdir(0) if err != nil { - return err + return errors.Wrap(err, "reading directory") } for _, fi := range fis { @@ -108,30 +116,35 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) + h.Logger.Printf("opening index: %s", filepath.Base(fi.Name())) index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { - return err + return errors.Wrap(err, "opening index") } if err := index.Open(); err != nil { if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err) + h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue } return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) } + h.mu.Lock() h.indexes[index.Name()] = index + h.mu.Unlock() } + h.Logger.Printf("open holder: complete") // Periodically flush cache. h.wg.Add(1) go func() { defer h.wg.Done(); h.monitorCacheFlush() }() h.Stats.Open() + + close(h.opened) return nil } @@ -145,12 +158,48 @@ func (h *Holder) Close() error { for _, index := range h.indexes { if err := index.Close(); err != nil { - return err + return errors.Wrap(err, "closing index") } } return nil } +// HasData returns true if Holder contains at least one index. +// This is used to determine if the rebalancing of data is necessary +// when a node joins the cluster. +func (h *Holder) HasData() (bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.indexes) > 0 { + return true, nil + } + // Open path to read all index directories. + if _, err := os.Stat(h.Path); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, errors.Wrap(err, "statting data dir") + } + + f, err := os.Open(h.Path) + if err != nil { + return false, errors.Wrap(err, "opening data dir") + } + defer f.Close() + + fis, err := f.Readdir(0) + if err != nil { + return false, errors.Wrap(err, "reading data dir") + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + return true, nil + } + return false, nil +} + // MaxSlices returns MaxSlice map for all indexes. func (h *Holder) MaxSlices() map[string]uint64 { a := make(map[string]uint64) @@ -169,13 +218,13 @@ func (h *Holder) MaxInverseSlices() map[string]uint64 { return a } -// Schema returns schema data for all indexes and frames. +// Schema returns schema information for all indexes, frames, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} for _, frame := range index.Frames() { - fi := &FrameInfo{Name: frame.Name()} + fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()} for _, view := range frame.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) } @@ -189,6 +238,50 @@ func (h *Holder) Schema() []*IndexInfo { return a } +// ApplySchema applies an internal Schema to Holder. +func (h *Holder) ApplySchema(schema *internal.Schema) error { + // Create indexes that don't exist. + for _, index := range schema.Indexes { + opt := IndexOptions{} + idx, err := h.CreateIndexIfNotExists(index.Name, opt) + if err != nil { + return errors.Wrap(err, "creating index") + } + // Create frames that don't exist. + for _, f := range index.Frames { + opt := decodeFrameOptions(f.Meta) + frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + if err != nil { + return errors.Wrap(err, "creating frame") + } + // Create views that don't exist. + for _, v := range f.Views { + _, err := frame.CreateViewIfNotExists(v) + if err != nil { + return errors.Wrap(err, "creating view") + } + } + } + // TODO: Create inputDefinitions that don't exist. + } + return nil +} + +// EncodeMaxSlices creates and internal representation of max slices. +func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { + return &internal.MaxSlices{ + Standard: h.MaxSlices(), + Inverse: h.MaxInverseSlices(), + } +} + +// EncodeSchema creates an internal representation of schema. +func (h *Holder) EncodeSchema() *internal.Schema { + return &internal.Schema{ + Indexes: EncodeIndexes(h.Indexes()), + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } @@ -254,16 +347,14 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { // Otherwise create a new index. index, err := h.newIndex(h.IndexPath(name), name) if err != nil { - return nil, err + return nil, errors.Wrap(err, "creating") } if err := index.Open(); err != nil { - return nil, err + return nil, errors.Wrap(err, "opening") } // Update options. - index.SetColumnLabel(opt.ColumnLabel) - index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index @@ -275,9 +366,11 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.LogOutput = h.LogOutput + index.Logger = h.Logger index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.Broadcaster + index.NewAttrStore = h.NewAttrStore + index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data")) return index, nil } @@ -294,12 +387,12 @@ func (h *Holder) DeleteIndex(name string) error { // Close index. if err := index.Close(); err != nil { - return err + return errors.Wrap(err, "closing") } // Delete index directory. if err := os.RemoveAll(h.IndexPath(name)); err != nil { - return err + return errors.Wrap(err, "removing directory") } // Remove reference. @@ -363,7 +456,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) } } } @@ -387,7 +480,7 @@ func (h *Holder) setFileLimit() { newLimit := &syscall.Rlimit{} if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) return } // If the soft limit is lower than the FileLimit constant, we will try to change it. @@ -411,42 +504,68 @@ func (h *Holder) setFileLimit() { } // Try setting again with lowered Max (hard limit) if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } // If we weren't trying to change the hard limit, let the user know something is wrong. } else { - h.logger().Printf("ERROR setting open file limit: %s", err) + h.Logger.Printf("ERROR setting open file limit: %s", err) } } // Check the limit after setting it. OS may not obey Setrlimit call. if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { - h.logger().Printf("ERROR checking open file limit: %s", err) + h.Logger.Printf("ERROR checking open file limit: %s", err) } else { if oldLimit.Cur < FileLimit { - h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) + h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) } } } } -func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } - -func (h *Holder) loadLocalID() error { - idPath := path.Join(h.Path, "ID") - localID := "" - localIDBytes, err := ioutil.ReadFile(idPath) - if err == nil { - localID = strings.TrimSpace(string(localIDBytes)) - } else { - u := uuid.NewV4() - localID = u.String() - err = ioutil.WriteFile(idPath, []byte(localID), 0600) - if err != nil { - return err - } +func (h *Holder) loadNodeID() (string, error) { + idPath := path.Join(h.Path, ".id") + nodeID := "" + h.Logger.Printf("load NodeID: %s", idPath) + if err := os.MkdirAll(h.Path, 0777); err != nil { + return "", errors.Wrap(err, "creating directory") } - h.LocalID = localID + + nodeIDBytes, err := ioutil.ReadFile(idPath) + if err == nil { + nodeID = strings.TrimSpace(string(nodeIDBytes)) + } else if os.IsNotExist(err) { + nodeID = uuid.NewV4().String() + err = ioutil.WriteFile(idPath, []byte(nodeID), 0600) + if err != nil { + return "", errors.Wrap(err, "writing file") + } + } else if err != nil { + return "", errors.Wrap(err, "reading file") + } + + return nodeID, nil +} + +// Log startup time and version to $DATA_DIR/.startup.log +func (h *Holder) logStartup() error { + time, err := time.Now().MarshalText() + if err != nil { + return errors.Wrap(err, "creating timestamp") + } + logLine := fmt.Sprintf("%s\t%s\n", time, Version) + + f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return errors.Wrap(err, "opening startup log") + } + + defer f.Close() + + if _, err = f.WriteString(logLine); err != nil { + return errors.Wrap(err, "writing startup log") + } + return nil } @@ -455,7 +574,7 @@ func (h *Holder) loadLocalID() error { type HolderSyncer struct { Holder *Holder - URI *URI + Node *Node Cluster *Cluster RemoteClient *http.Client @@ -511,7 +630,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.URI.HostPort(), di.Name, slice) { + if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { continue } @@ -548,36 +667,33 @@ func (s *HolderSyncer) syncIndex(index string) error { // Read block checksums. blks, err := idx.ColumnAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "getting blocks") } s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) - if err != nil { - return err - } + for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. m, err := client.ColumnAttrDiff(context.Background(), index, blks) if err != nil { - return err + return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.Host}) + s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID}) // Update local copy. if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil { - return err + return errors.Wrap(err, "setting attrs") } // Recompute blocks. blks, err = idx.ColumnAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "recomputing blocks") } } @@ -586,7 +702,7 @@ func (s *HolderSyncer) syncIndex(index string) error { // syncFrame synchronizes frame attributes with the rest of the cluster. func (s *HolderSyncer) syncFrame(index, name string) error { - // Retrieve index reference. + // Retrieve frame reference. f := s.Holder.Frame(index, name) if f == nil { return nil @@ -597,16 +713,13 @@ func (s *HolderSyncer) syncFrame(index, name string) error { // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "getting blocks") } s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) // Sync with every other host. - for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) { - client, err := NewInternalHTTPClient(node.Host, s.RemoteClient) - if err != nil { - return err - } + for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { + client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. @@ -614,21 +727,21 @@ func (s *HolderSyncer) syncFrame(index, name string) error { if err == ErrFrameNotFound { continue // frame not created remotely yet, skip } else if err != nil { - return err + return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.Host}) + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { - return err + return errors.Wrap(err, "setting attrs") } // Recompute blocks. blks, err = f.RowAttrStore().Blocks() if err != nil { - return err + return errors.Wrap(err, "recomputing blocks") } } @@ -646,26 +759,88 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Ensure view exists locally. v, err := f.CreateViewIfNotExists(view) if err != nil { - return err + return errors.Wrap(err, "creating view") } // Ensure fragment exists locally. frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { - return err + return errors.Wrap(err, "creating fragment") } // Sync fragments together. fs := FragmentSyncer{ Fragment: frag, - Host: s.URI.HostPort(), + Node: s.Node, Cluster: s.Cluster, Closing: s.Closing, RemoteClient: s.RemoteClient, } if err := fs.SyncFragment(); err != nil { - return err + return errors.Wrap(err, "syncing fragment") } return nil } + +// HolderCleaner removes fragments and data files that are no longer used. +type HolderCleaner struct { + Node *Node + + Holder *Holder + Cluster *Cluster + + // Signals that the sync should stop. + Closing <-chan struct{} +} + +// IsClosing returns true if the cleaner has been marked to close. +func (c *HolderCleaner) IsClosing() bool { + select { + case <-c.Closing: + return true + default: + return false + } +} + +// CleanHolder compares the holder with the cluster state and removes +// any unnecessary fragments and files. +func (c *HolderCleaner) CleanHolder() error { + for _, index := range c.Holder.Indexes() { + // Verify cleaner has not closed. + if c.IsClosing() { + return nil + } + + // Get the fragments that node is responsible for (based on hash(index, node)). + containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) + + // Get the fragments registered in memory. + for _, frame := range index.Frames() { + for _, view := range frame.Views() { + for _, fragment := range view.Fragments() { + fragSlice := fragment.Slice() + // Ignore fragments that should be present. + if uint64InSlice(fragSlice, containedSlices) { + continue + } + // Delete fragment. + if err := view.DeleteFragment(fragSlice); err != nil { + return errors.Wrap(err, "deleting fragment") + } + } + } + } + } + return nil +} + +func uint64InSlice(i uint64, s []uint64) bool { + for _, o := range s { + if i == o { + return true + } + } + return false +} diff --git a/holder_test.go b/holder_test.go index 2bf656e10..e819c6e94 100644 --- a/holder_test.go +++ b/holder_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "bytes" "context" "os" "path/filepath" @@ -24,12 +25,17 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) func TestHolder_Open(t *testing.T) { t.Run("ErrIndexName", func(t *testing.T) { h := test.MustOpenHolder() + + bufLogger := test.NewBufferLogger() + h.Holder.Logger = bufLogger + defer h.Close() if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { @@ -39,8 +45,12 @@ func TestHolder_Open(t *testing.T) { } if err := h.Reopen(); err != nil { t.Fatal(err) - } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { - t.Fatalf("expected log error:\n%s", logOutput) + } + + if bufbytes, err := bufLogger.ReadAll(); err != nil { + t.Fatal(err) + } else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) { + t.Fatalf("expected log error:\n%s", bufbytes) } }) @@ -64,22 +74,6 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() - defer h.Close() - - if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { - t.Fatal(err) - } - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { - t.Fatalf("unexpected error: %s", err) - } - }) t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() @@ -284,6 +278,53 @@ func TestHolder_Open(t *testing.T) { }) } +func TestHolder_HasData(t *testing.T) { + t.Run("IndexDirectory", func(t *testing.T) { + h := test.MustOpenHolder() + defer h.Close() + + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) + } + + if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, but ", ok, err) + } + }) + + t.Run("Peek", func(t *testing.T) { + h := test.NewHolder() + + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) + } + + // Create an index directory to indicate data exists. + if err := os.Mkdir(h.IndexPath("test"), 0777); err != nil { + t.Fatal(err) + } + + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, no err, but", ok, err) + } + }) + + t.Run("Peek at missing directory", func(t *testing.T) { + h := test.NewHolder() + + // Ensure that hasData is false when dir doesn't exist. + h.Path = "bad-path" + + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) + } + }) +} + // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { hldr := test.MustOpenHolder() @@ -320,7 +361,7 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) - client := pilosa.GetHTTPClient(nil) + client := server.GetHTTPClient(nil) // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -330,20 +371,25 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer hldr1.Close() s := test.NewServer() defer s.Close() - s.Handler.Holder = hldr1.Holder + s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder - e.Scheme = cluster.Nodes[1].Scheme - e.Host = cluster.Nodes[1].Host + e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].Host = "localhost:0" - cluster.Nodes[1].Host = test.MustParseURLHost(s.URL) + + uri, err := pilosa.NewURIFromAddress(s.URL) + if err != nil { + t.Fatal(err) + } + + cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[1].URI = *uri // Create frames on nodes. for _, hldr := range []*test.Holder{hldr0, hldr1} { @@ -395,15 +441,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.Index("y").SetRemoteMaxSlice(3) // Set up syncer. - uri, err := cluster.Nodes[0].URI() - if err != nil { - t.Fatal(err) - } syncer := pilosa.HolderSyncer{ Holder: hldr0.Holder, - URI: uri, + Node: cluster.Nodes[0], Cluster: cluster, - RemoteClient: pilosa.GetHTTPClient(nil), + RemoteClient: server.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, } @@ -440,3 +482,141 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } } } + +// Ensure holder can clean up orphaned fragments. +func TestHolderCleaner_CleanHolder(t *testing.T) { + cluster := test.NewCluster(2) + + // Create a local holder. + hldr0 := test.MustOpenHolder() + defer hldr0.Close() + + // Mock 2-node, fully replicated cluster. + cluster.ReplicaN = 2 + + cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + + // Create frames on nodes. + for _, hldr := range []*test.Holder{hldr0} { + hldr.MustCreateFrameIfNotExists("i", "f") + hldr.MustCreateFrameIfNotExists("i", "f0") + hldr.MustCreateFrameIfNotExists("y", "z") + } + + // Set data on the local holder. + f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) + if _, err := f.SetBit(0, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 4000); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(2, 20); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(3, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(120, 10); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(200, 4); err != nil { + t.Fatal(err) + } + + f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) + if _, err := f.SetBit(9, SliceWidth+5); err != nil { + t.Fatal(err) + } + + f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2) + if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil { + t.Fatal(err) + } + + // Set highest slice. + hldr0.Index("i").SetRemoteMaxSlice(1) + hldr0.Index("y").SetRemoteMaxSlice(2) + + // Keep replication the same and ensure we get the expected results. + cluster.ReplicaN = 2 + + // Set up cleaner for replication 2. + cleaner2 := pilosa.HolderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner2.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*test.Holder{hldr0} { + f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected bits(%d/0): %+v", i, a) + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected bits(%d/2): %+v", i, a) + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/3): %+v", i, a) + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/120): %+v", i, a) + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected bits(%d/200): %+v", i, a) + } + + f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + a := f.Row(9).Bits() + if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + } + if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + } + f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + } + } + + // Change replication factor to ensure we have fragments to remove. + cluster.ReplicaN = 1 + + // Set up cleaner for replication 1. + cleaner1 := pilosa.HolderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner1.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*test.Holder{hldr0} { + f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected bits(%d/0): %+v", i, a) + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected bits(%d/2): %+v", i, a) + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/3): %+v", i, a) + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected bits(%d/120): %+v", i, a) + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected bits(%d/200): %+v", i, a) + } + + f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + if f != nil { + t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) + } + + f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + } + } +} diff --git a/index.go b/index.go index bdf2832b0..dc676c304 100644 --- a/index.go +++ b/index.go @@ -17,7 +17,6 @@ package pilosa import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -31,7 +30,6 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "columnID" InputDefinitionDir = ".input-definitions" ) @@ -41,13 +39,6 @@ type Index struct { path string name string - // Default time quantum for all frames in index. - // This can be overridden by individual frames. - timeQuantum TimeQuantum - - // Label used for referring to columns in index. - columnLabel string - // Frames by name. frames map[string]*Frame @@ -55,8 +46,10 @@ type Index struct { remoteMaxSlice uint64 remoteMaxInverseSlice uint64 + NewAttrStore func(string) AttrStore + // Column attribute storage and cache. - columnAttrStore *AttrStore + columnAttrStore AttrStore // InputDefinitions by name. inputDefinitions map[string]*InputDefinition @@ -64,7 +57,7 @@ type Index struct { broadcaster Broadcaster Stats StatsClient - LogOutput io.Writer + Logger Logger } // NewIndex returns a new instance of Index. @@ -83,13 +76,12 @@ func NewIndex(path, name string) (*Index, error) { remoteMaxSlice: 0, remoteMaxInverseSlice: 0, - columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), - - columnLabel: DefaultColumnLabel, + NewAttrStore: NewNopAttrStore, + columnAttrStore: NopAttrStore, broadcaster: NopBroadcaster, Stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, }, nil } @@ -100,40 +92,7 @@ func (i *Index) Name() string { return i.name } func (i *Index) Path() string { return i.path } // ColumnAttrStore returns the storage for column attributes. -func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore } - -// SetColumnLabel sets the column label. Persists to meta file on update. -func (i *Index) SetColumnLabel(v string) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || i.columnLabel == v { - return nil - } - - // Make sure columnLabel is valid name - err := ValidateLabel(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - i.columnLabel = v - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - -// ColumnLabel returns the column label. -func (i *Index) ColumnLabel() string { - i.mu.RLock() - v := i.columnLabel - i.mu.RUnlock() - return v -} +func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } // Options returns all options for this index. func (i *Index) Options() IndexOptions { @@ -143,10 +102,7 @@ func (i *Index) Options() IndexOptions { } func (i *Index) options() IndexOptions { - return IndexOptions{ - ColumnLabel: i.columnLabel, - TimeQuantum: i.timeQuantum, - } + return IndexOptions{} } // Open opens and initializes the index. @@ -213,8 +169,6 @@ func (i *Index) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) if os.IsNotExist(err) { - i.timeQuantum = "" - i.columnLabel = DefaultColumnLabel return nil } else if err != nil { return err @@ -225,19 +179,18 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. - i.timeQuantum = TimeQuantum(pb.TimeQuantum) - i.columnLabel = pb.ColumnLabel return nil } +// NOTE: Until we introduce new attributes to store in the index .meta file, +// we don't need to actually write the file. The code related to index.options +// and the index meta file are left in place for future use. +/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{ - TimeQuantum: string(i.timeQuantum), - ColumnLabel: i.columnLabel, - }) + buf, err := proto.Marshal(&internal.IndexMeta{}) if err != nil { return err } @@ -249,6 +202,7 @@ func (i *Index) saveMeta() error { return nil } +*/ // Close closes the index and its frames. func (i *Index) Close() error { @@ -256,9 +210,7 @@ func (i *Index) Close() error { defer i.mu.Unlock() // Close the attribute store. - if i.columnAttrStore != nil { - i.columnAttrStore.Close() - } + i.columnAttrStore.Close() // Close all frames. for _, f := range i.frames { @@ -321,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) { i.remoteMaxInverseSlice = v } -// TimeQuantum returns the default time quantum for the index. -func (i *Index) TimeQuantum() TimeQuantum { - i.mu.RLock() - defer i.mu.RUnlock() - return i.timeQuantum -} - -// SetTimeQuantum sets the default time quantum for the index. -func (i *Index) SetTimeQuantum(q TimeQuantum) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on index. - i.timeQuantum = q - - // Perist meta data to disk. - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } @@ -392,6 +316,20 @@ func (i *Index) Frames() []*Frame { return a } +// InputDefinitions returns a list of all inputDefinitions in the index. +func (i *Index) InputDefinitions() []*InputDefinition { + i.mu.RLock() + defer i.mu.RUnlock() + + a := make([]*InputDefinition, 0, len(i.inputDefinitions)) + for _, d := range i.inputDefinitions { + a = append(a, d) + } + //sort.Sort(inputDefintionSlice(a)) // TODO + + return a +} + // RecalculateCaches recalculates caches on every frame in the index. func (i *Index) RecalculateCaches() { for _, frame := range i.Frames() { @@ -431,22 +369,9 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate that row label does not match column label. - if i.columnLabel == opt.RowLabel || (opt.RowLabel == "" && i.columnLabel == DefaultRowLabel) { - return nil, ErrColumnRowLabelEqual - } - // Validate mutually exclusive options if ranges are enabled. if opt.RangeEnabled { - if opt.InverseEnabled { - return nil, ErrInverseRangeNotAllowed - } else if opt.CacheType != "" && opt.CacheType != CacheTypeNone { - return nil, ErrRangeCacheNotAllowed - } - } else { - if len(opt.Fields) > 0 { - return nil, ErrFrameFieldsNotAllowed - } + i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame") } // Validate fields. @@ -467,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } - // Default the time quantum to what is set on the Index. - timeQuantum := i.timeQuantum - if opt.TimeQuantum != "" { - timeQuantum = opt.TimeQuantum - } - if err := f.SetTimeQuantum(timeQuantum); err != nil { + // Set the time quantum. + if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() return nil, err } @@ -483,33 +404,20 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } f.cacheType = opt.CacheType - // Set options. - if opt.RowLabel != "" { - f.rowLabel = opt.RowLabel - } if opt.CacheSize != 0 { f.cacheSize = opt.CacheSize } f.inverseEnabled = opt.InverseEnabled - f.rangeEnabled = opt.RangeEnabled + + // Set fields. + f.fields = opt.Fields if err := f.saveMeta(); err != nil { f.Close() return nil, err } - f.rangeEnabled = opt.RangeEnabled - - // Set schema & save. - f.schema = &FrameSchema{ - Fields: opt.Fields, - } - if err := f.saveSchema(); err != nil { - f.Close() - return nil, err - } - // Add to index's frame lookup. i.frames[name] = f @@ -521,9 +429,10 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { if err != nil { return nil, err } - f.LogOutput = i.LogOutput + f.Logger = i.Logger f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) f.broadcaster = i.broadcaster + f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) return f, nil } @@ -572,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// MergeSchemas combines indexes and frames from a and b into one schema. -func MergeSchemas(a, b []*IndexInfo) []*IndexInfo { - // Generate a map from both schemas. - m := make(map[string]map[string]map[string]struct{}) - for _, idxs := range [][]*IndexInfo{a, b} { - for _, idx := range idxs { - if m[idx.Name] == nil { - m[idx.Name] = make(map[string]map[string]struct{}) - } - for _, frame := range idx.Frames { - if m[idx.Name][frame.Name] == nil { - m[idx.Name][frame.Name] = make(map[string]struct{}) - } - for _, view := range frame.Views { - m[idx.Name][frame.Name][view.Name] = struct{}{} - } - } - } - } - - // Generate new schema from map. - idxs := make([]*IndexInfo, 0, len(m)) - for idx, frames := range m { - di := &IndexInfo{Name: idx} - for frame, views := range frames { - fi := &FrameInfo{Name: frame} - for view := range views { - fi.Views = append(fi.Views, &ViewInfo{Name: view}) - } - sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) - } - sort.Sort(frameInfoSlice(di.Frames)) - idxs = append(idxs, di) - } - sort.Sort(indexInfoSlice(idxs)) - - return idxs -} - // EncodeIndexes converts a into its internal representation. func EncodeIndexes(a []*Index) []*internal.Index { other := make([]*internal.Index, len(a)) @@ -623,27 +492,19 @@ func EncodeIndexes(a []*Index) []*internal.Index { // encodeIndex converts d into its internal representation. func encodeIndex(d *Index) *internal.Index { - io := d.options() return &internal.Index{ - Name: d.name, - Meta: io.Encode(), - MaxSlice: d.MaxSlice(), - Frames: encodeFrames(d.Frames()), + Name: d.name, + Frames: encodeFrames(d.Frames()), + InputDefinitions: encodeInputDefinitions(d.InputDefinitions()), } } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` -} +type IndexOptions struct{} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - ColumnLabel: i.ColumnLabel, - TimeQuantum: string(i.TimeQuantum), - } + return &internal.IndexMeta{} } // hasTime returns true if a contains a non-nil time. @@ -688,7 +549,6 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin for _, fr := range pb.Frames { opt := FrameOptions{ // Deprecating row labels per #810. So, setting the default row label here. - RowLabel: DefaultRowLabel, InverseEnabled: fr.Meta.InverseEnabled, CacheType: fr.Meta.CacheType, CacheSize: fr.Meta.CacheSize, @@ -723,7 +583,6 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { if err != nil { return nil, err } - inputDef.broadcaster = i.broadcaster return inputDef, nil } @@ -776,7 +635,6 @@ func (i *Index) openInputDefinitions() error { return nil } } - } return nil } @@ -784,7 +642,8 @@ func (i *Index) openInputDefinitions() error { // InputBits Process the []Bit though the Frame import process func (i *Index) InputBits(frame string, bits []*Bit) error { var rowIDs, columnIDs []uint64 - timestamps := make([]*time.Time, len(bits)) + var timestamps []*time.Time + f := i.Frame(frame) if f == nil { return fmt.Errorf("Frame not found: %s", frame) @@ -799,6 +658,11 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { // Convert timestamps to time.Time. if bit.Timestamp > 0 { + // Don't create a full timestamps slice unless + // at least one bit contains a timestamp. + if len(timestamps) == 0 { + timestamps = make([]*time.Time, len(bits)) + } t := time.Unix(bit.Timestamp, 0) timestamps[i] = &t } diff --git a/index_test.go b/index_test.go index 5e44a53a4..b0a5b3506 100644 --- a/index_test.go +++ b/index_test.go @@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - // Create frame with explicit quantum. f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) if err != nil { @@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) { t.Fatalf("unexpected frame time quantum: %s", q) } }) - - t.Run("Inherited", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - - // Create frame. - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { - t.Fatalf("unexpected frame time quantum: %s", q) - } - }) }) // Ensure frame can include range columns. @@ -99,55 +76,81 @@ func TestIndex_CreateFrame(t *testing.T) { // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: false, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, }, }); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + } else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, }) { - t.Fatalf("unexpected schema: %#v", f.Schema()) + t.Fatalf("unexpected fields: %#v", f.Fields()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + } else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, }) { - t.Fatalf("unexpected schema after reopen: %#v", f.Schema()) + t.Fatalf("unexpected fields after reopen: %#v", f.Fields()) } }) - t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + t.Run("ErrInverseRangeAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - InverseEnabled: true, + frame, err := index.CreateFrame("f", pilosa.FrameOptions{ RangeEnabled: true, - }); err != pilosa.ErrInverseRangeNotAllowed { + InverseEnabled: true, + Fields: []*pilosa.Field{ + &pilosa.Field{ + Name: "myfield", + Type: pilosa.FieldTypeInt, + Min: -20, + Max: 100, + }, + }, + }) + if err != nil { t.Fatal(err) } + + ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetFieldValue(1, "myfield", 87) + if !ch || err != nil { + t.Fatal(ch, err) + } + views := frame.Views() + if len(views) != 3 { + var names string + for _, v := range views { + names = names + v.Name() + " " + } + t.Fatalf("Unexpected views: %s", names) + } + }) - t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) { + t.Run("ErrRangeCacheAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeRanked, - }); err != pilosa.ErrRangeCacheNotAllowed { + CacheType: pilosa.CacheTypeRanked, + }); err != nil { t.Fatal(err) } }) @@ -156,15 +159,14 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), }); err != nil { t.Fatal(err) } }) - t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() @@ -172,7 +174,7 @@ func TestIndex_CreateFrame(t *testing.T) { Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt}, }, - }); err != pilosa.ErrFrameFieldsNotAllowed { + }); err != nil { t.Fatal(err) } }) @@ -182,7 +184,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "", Type: pilosa.FieldTypeInt}, }, @@ -196,7 +197,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: "bad_type"}, }, @@ -210,7 +210,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, @@ -219,32 +219,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) }) - - // Ensure frame cannot be created with a matching row label. - t.Run("ErrColumnRowLabelEqual", func(t *testing.T) { - t.Run("Explicit", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - _, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("Default", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil { - t.Fatal(err) - } - - _, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != pilosa.ErrColumnRowLabelEqual { - t.Fatalf("unexpected error: %s", err) - } - }) - }) } // Ensure index can delete a frame. @@ -270,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) { } } -// Ensure index can set the default time quantum. -func TestIndex_SetTimeQuantum(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set & retrieve time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload index and verify that it is persisted. - if err := index.Reopen(); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - // Ensure index can delete a frame. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") @@ -307,7 +261,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -334,7 +288,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -354,7 +308,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -385,7 +339,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -407,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - err := index.InputBits("f", bits) if !strings.Contains(err.Error(), "Frame not found") { t.Fatalf("Expected Frame not found error, actual error: %s", err) } // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil { t.Fatal(err) } diff --git a/input_definition.go b/input_definition.go index 84d22d4ac..b18b4980e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -36,12 +36,11 @@ var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleR // InputDefinition represents a container for the data input definition. type InputDefinition struct { - name string - path string - index string - broadcaster Broadcaster - frames []InputFrame - fields []InputDefinitionField + name string + path string + index string + frames []InputFrame + fields []InputDefinitionField } // NewInputDefinition returns a new instance of InputDefinition. @@ -86,17 +85,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name for _, fr := range pb.Frames { - frameMeta := fr.Meta inputFrame := InputFrame{ - Name: fr.Name, - Options: FrameOptions{ - // Deprecating row labels per #810. So, setting the default row label here. - RowLabel: DefaultRowLabel, - InverseEnabled: frameMeta.InverseEnabled, - CacheSize: frameMeta.CacheSize, - CacheType: frameMeta.CacheType, - TimeQuantum: TimeQuantum(frameMeta.TimeQuantum), - }, + Name: fr.Name, + Options: *decodeFrameOptions(fr.Meta), } i.frames = append(i.frames, inputFrame) } @@ -290,7 +281,7 @@ func (i *InputDefinitionInfo) Validate() error { } } - // Validate columnLabel and duplicate primaryKey. + // Validate duplicate primaryKey. for _, field := range i.Fields { if field.Name == "" { return ErrInputDefinitionNameRequired @@ -338,6 +329,43 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { return &def } +// encodeInputDefinitions converts a into its internal representation. +func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition { + other := make([]*internal.InputDefinition, len(a)) + for i := range a { + other[i] = encodeInputDefinition(a[i]) + } + return other +} + +// encodeInputDefinition converts i into its internal representation. +func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition { + //fo := f.options() + return &internal.InputDefinition{ + Name: i.name, + Frames: encodeInputFrames(i.frames), + Fields: encodeInputDefinitionFields(i.fields), + } +} + +// encodeInputFrames converts a into its internal representation. +func encodeInputFrames(a []InputFrame) []*internal.Frame { + other := make([]*internal.Frame, len(a)) + for i := range a { + other[i] = a[i].Encode() + } + return other +} + +// encodeInputDefinitionFields converts a into its internal representation. +func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField { + other := make([]*internal.InputDefinitionField, len(a)) + for i := range a { + other[i] = a[i].Encode() + } + return other +} + // AddFrame manually add frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) diff --git a/input_definition_test.go b/input_definition_test.go index 93d0e393d..efd13bec3 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -30,7 +30,7 @@ func TestInputDefinition_Open(t *testing.T) { defer index.Close() // Create Input Definition. - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} @@ -115,14 +115,14 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } - frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}} action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} @@ -131,7 +131,7 @@ func TestActionValidation(t *testing.T) { t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } - frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}} action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1c26aa307..2e4d39239 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -15,24 +15,38 @@ BlockDataRequest BlockDataResponse Cache - MaxSlicesResponse + MaxSlices CreateSliceMessage DeleteIndexMessage CreateIndexMessage CreateFrameMessage DeleteFrameMessage + CreateFieldMessage + DeleteFieldMessage Frame + Schema Index InputDefinition InputDefinitionField InputDefinitionAction CreateInputDefinitionMessage DeleteInputDefinitionMessage + URI + Node + NodeStateMessage + NodeEventMessage NodeStatus ClusterStatus - FrameSchema Field + CreateViewMessage DeleteViewMessage + ResizeInstruction + ResizeSource + ResizeInstructionComplete + SetCoordinatorMessage + UpdateCoordinatorMessage + Topology + RecalculateCaches */ package internal @@ -54,8 +68,6 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` - TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -63,22 +75,7 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -func (m *IndexMeta) GetColumnLabel() string { - if m != nil { - return m.ColumnLabel - } - return "" -} - -func (m *IndexMeta) GetTimeQuantum() string { - if m != nil { - return m.TimeQuantum - } - return "" -} - type FrameMeta struct { - RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` @@ -92,13 +89,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetRowLabel() string { - if m != nil { - return m.RowLabel - } - return "" -} - func (m *FrameMeta) GetInverseEnabled() bool { if m != nil { return m.InverseEnabled @@ -245,18 +235,26 @@ func (m *Cache) GetIDs() []uint64 { return nil } -type MaxSlicesResponse struct { - MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +type MaxSlices struct { + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } -func (m *MaxSlicesResponse) String() string { return proto.CompactTextString(m) } -func (*MaxSlicesResponse) ProtoMessage() {} -func (*MaxSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } +func (m *MaxSlices) Reset() { *m = MaxSlices{} } +func (m *MaxSlices) String() string { return proto.CompactTextString(m) } +func (*MaxSlices) ProtoMessage() {} +func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } -func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { +func (m *MaxSlices) GetStandard() map[string]uint64 { if m != nil { - return m.MaxSlices + return m.Standard + } + return nil +} + +func (m *MaxSlices) GetInverse() map[string]uint64 { + if m != nil { + return m.Inverse } return nil } @@ -389,15 +387,80 @@ func (m *DeleteFrameMessage) GetFrame() string { return "" } +type CreateFieldMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field *Field `protobuf:"bytes,3,opt,name=Field" json:"Field,omitempty"` +} + +func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } +func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFieldMessage) ProtoMessage() {} +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } + +func (m *CreateFieldMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateFieldMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *CreateFieldMessage) GetField() *Field { + if m != nil { + return m.Field + } + return nil +} + +type DeleteFieldMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` +} + +func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } +func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFieldMessage) ProtoMessage() {} +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } + +func (m *DeleteFieldMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *DeleteFieldMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *DeleteFieldMessage) GetField() string { + if m != nil { + return m.Field + } + return "" +} + type Frame struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } func (m *Frame) String() string { return proto.CompactTextString(m) } func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Frame) GetName() string { if m != nil { @@ -413,19 +476,39 @@ func (m *Frame) GetMeta() *FrameMeta { return nil } +func (m *Frame) GetViews() []string { + if m != nil { + return m.Views + } + return nil +} + +type Schema struct { + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` +} + +func (m *Schema) Reset() { *m = Schema{} } +func (m *Schema) String() string { return proto.CompactTextString(m) } +func (*Schema) ProtoMessage() {} +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } + +func (m *Schema) GetIndexes() []*Index { + if m != nil { + return m.Indexes + } + return nil +} + type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` - Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"` InputDefinitions []*InputDefinition `protobuf:"bytes,6,rep,name=InputDefinitions" json:"InputDefinitions,omitempty"` } func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *Index) GetName() string { if m != nil { @@ -434,20 +517,6 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetMeta() *IndexMeta { - if m != nil { - return m.Meta - } - return nil -} - -func (m *Index) GetMaxSlice() uint64 { - if m != nil { - return m.MaxSlice - } - return 0 -} - func (m *Index) GetFrames() []*Frame { if m != nil { return m.Frames @@ -455,13 +524,6 @@ func (m *Index) GetFrames() []*Frame { return nil } -func (m *Index) GetSlices() []uint64 { - if m != nil { - return m.Slices - } - return nil -} - func (m *Index) GetInputDefinitions() []*InputDefinition { if m != nil { return m.InputDefinitions @@ -478,7 +540,7 @@ type InputDefinition struct { func (m *InputDefinition) Reset() { *m = InputDefinition{} } func (m *InputDefinition) String() string { return proto.CompactTextString(m) } func (*InputDefinition) ProtoMessage() {} -func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *InputDefinition) GetName() string { if m != nil { @@ -510,7 +572,7 @@ type InputDefinitionField struct { func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } func (m *InputDefinitionField) String() string { return proto.CompactTextString(m) } func (*InputDefinitionField) ProtoMessage() {} -func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *InputDefinitionField) GetName() string { if m != nil { @@ -543,7 +605,7 @@ type InputDefinitionAction struct { func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } func (*InputDefinitionAction) ProtoMessage() {} -func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *InputDefinitionAction) GetFrame() string { if m != nil { @@ -582,7 +644,7 @@ func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinit func (m *CreateInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*CreateInputDefinitionMessage) ProtoMessage() {} func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{17} + return fileDescriptorPrivate, []int{20} } func (m *CreateInputDefinitionMessage) GetIndex() string { @@ -608,7 +670,7 @@ func (m *DeleteInputDefinitionMessage) Reset() { *m = DeleteInputDefinit func (m *DeleteInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*DeleteInputDefinitionMessage) ProtoMessage() {} func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{18} + return fileDescriptorPrivate, []int{21} } func (m *DeleteInputDefinitionMessage) GetIndex() string { @@ -625,78 +687,182 @@ func (m *DeleteInputDefinitionMessage) GetName() string { return "" } -type NodeStatus struct { - Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` - Scheme string `protobuf:"bytes,4,opt,name=Scheme,proto3" json:"Scheme,omitempty"` +type URI struct { + Scheme string `protobuf:"bytes,1,opt,name=Scheme,proto3" json:"Scheme,omitempty"` + Host string `protobuf:"bytes,2,opt,name=Host,proto3" json:"Host,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *NodeStatus) Reset() { *m = NodeStatus{} } -func (m *NodeStatus) String() string { return proto.CompactTextString(m) } -func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (m *URI) Reset() { *m = URI{} } +func (m *URI) String() string { return proto.CompactTextString(m) } +func (*URI) ProtoMessage() {} +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } -func (m *NodeStatus) GetHost() string { - if m != nil { - return m.Host - } - return "" -} - -func (m *NodeStatus) GetState() string { - if m != nil { - return m.State - } - return "" -} - -func (m *NodeStatus) GetIndexes() []*Index { - if m != nil { - return m.Indexes - } - return nil -} - -func (m *NodeStatus) GetScheme() string { +func (m *URI) GetScheme() string { if m != nil { return m.Scheme } return "" } +func (m *URI) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *URI) GetPort() uint32 { + if m != nil { + return m.Port + } + return 0 +} + +type Node struct { + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` +} + +func (m *Node) Reset() { *m = Node{} } +func (m *Node) String() string { return proto.CompactTextString(m) } +func (*Node) ProtoMessage() {} +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } + +func (m *Node) GetID() string { + if m != nil { + return m.ID + } + return "" +} + +func (m *Node) GetURI() *URI { + if m != nil { + return m.URI + } + return nil +} + +func (m *Node) GetIsCoordinator() bool { + if m != nil { + return m.IsCoordinator + } + return false +} + +type NodeStateMessage struct { + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } +func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } +func (*NodeStateMessage) ProtoMessage() {} +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } + +func (m *NodeStateMessage) GetNodeID() string { + if m != nil { + return m.NodeID + } + return "" +} + +func (m *NodeStateMessage) GetState() string { + if m != nil { + return m.State + } + return "" +} + +type NodeEventMessage struct { + Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` +} + +func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } +func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } +func (*NodeEventMessage) ProtoMessage() {} +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } + +func (m *NodeEventMessage) GetEvent() uint32 { + if m != nil { + return m.Event + } + return 0 +} + +func (m *NodeEventMessage) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + +type NodeStatus struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` +} + +func (m *NodeStatus) Reset() { *m = NodeStatus{} } +func (m *NodeStatus) String() string { return proto.CompactTextString(m) } +func (*NodeStatus) ProtoMessage() {} +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } + +func (m *NodeStatus) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + +func (m *NodeStatus) GetMaxSlices() *MaxSlices { + if m != nil { + return m.MaxSlices + } + return nil +} + +func (m *NodeStatus) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type ClusterStatus struct { - Nodes []*NodeStatus `protobuf:"bytes,1,rep,name=Nodes" json:"Nodes,omitempty"` + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` } func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } -func (m *ClusterStatus) GetNodes() []*NodeStatus { +func (m *ClusterStatus) GetClusterID() string { + if m != nil { + return m.ClusterID + } + return "" +} + +func (m *ClusterStatus) GetState() string { + if m != nil { + return m.State + } + return "" +} + +func (m *ClusterStatus) GetNodes() []*Node { if m != nil { return m.Nodes } return nil } -type FrameSchema struct { - Fields []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"` -} - -func (m *FrameSchema) Reset() { *m = FrameSchema{} } -func (m *FrameSchema) String() string { return proto.CompactTextString(m) } -func (*FrameSchema) ProtoMessage() {} -func (*FrameSchema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } - -func (m *FrameSchema) GetFields() []*Field { - if m != nil { - return m.Fields - } - return nil -} - type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -707,7 +873,7 @@ type Field struct { func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *Field) GetName() string { if m != nil { @@ -737,6 +903,38 @@ func (m *Field) GetMax() int64 { return 0 } +type CreateViewMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` +} + +func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } +func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } +func (*CreateViewMessage) ProtoMessage() {} +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } + +func (m *CreateViewMessage) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *CreateViewMessage) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *CreateViewMessage) GetView() string { + if m != nil { + return m.View + } + return "" +} + type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -746,7 +944,7 @@ type DeleteViewMessage struct { func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -769,6 +967,208 @@ func (m *DeleteViewMessage) GetView() string { return "" } +type ResizeInstruction struct { + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` + Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` +} + +func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } +func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } +func (*ResizeInstruction) ProtoMessage() {} +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } + +func (m *ResizeInstruction) GetJobID() int64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ResizeInstruction) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + +func (m *ResizeInstruction) GetCoordinator() *Node { + if m != nil { + return m.Coordinator + } + return nil +} + +func (m *ResizeInstruction) GetSources() []*ResizeSource { + if m != nil { + return m.Sources + } + return nil +} + +func (m *ResizeInstruction) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + +func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { + if m != nil { + return m.ClusterStatus + } + return nil +} + +type ResizeSource struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` +} + +func (m *ResizeSource) Reset() { *m = ResizeSource{} } +func (m *ResizeSource) String() string { return proto.CompactTextString(m) } +func (*ResizeSource) ProtoMessage() {} +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } + +func (m *ResizeSource) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + +func (m *ResizeSource) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *ResizeSource) GetFrame() string { + if m != nil { + return m.Frame + } + return "" +} + +func (m *ResizeSource) GetView() string { + if m != nil { + return m.View + } + return "" +} + +func (m *ResizeSource) GetSlice() uint64 { + if m != nil { + return m.Slice + } + return 0 +} + +type ResizeInstructionComplete struct { + JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` +} + +func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComplete{} } +func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } +func (*ResizeInstructionComplete) ProtoMessage() {} +func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { + return fileDescriptorPrivate, []int{33} +} + +func (m *ResizeInstructionComplete) GetJobID() int64 { + if m != nil { + return m.JobID + } + return 0 +} + +func (m *ResizeInstructionComplete) GetNode() *Node { + if m != nil { + return m.Node + } + return nil +} + +func (m *ResizeInstructionComplete) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +type SetCoordinatorMessage struct { + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` +} + +func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } +func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*SetCoordinatorMessage) ProtoMessage() {} +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{34} } + +func (m *SetCoordinatorMessage) GetNew() *Node { + if m != nil { + return m.New + } + return nil +} + +type UpdateCoordinatorMessage struct { + New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` +} + +func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } +func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } +func (*UpdateCoordinatorMessage) ProtoMessage() {} +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{35} } + +func (m *UpdateCoordinatorMessage) GetNew() *Node { + if m != nil { + return m.New + } + return nil +} + +type Topology struct { + ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` +} + +func (m *Topology) Reset() { *m = Topology{} } +func (m *Topology) String() string { return proto.CompactTextString(m) } +func (*Topology) ProtoMessage() {} +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{36} } + +func (m *Topology) GetClusterID() string { + if m != nil { + return m.ClusterID + } + return "" +} + +func (m *Topology) GetNodeIDs() []string { + if m != nil { + return m.NodeIDs + } + return nil +} + +type RecalculateCaches struct { +} + +func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } +func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } +func (*RecalculateCaches) ProtoMessage() {} +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{37} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -776,24 +1176,38 @@ func init() { proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") + proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices") proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") + proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") + proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") proto.RegisterType((*InputDefinitionAction)(nil), "internal.InputDefinitionAction") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") + proto.RegisterType((*URI)(nil), "internal.URI") + proto.RegisterType((*Node)(nil), "internal.Node") + proto.RegisterType((*NodeStateMessage)(nil), "internal.NodeStateMessage") + proto.RegisterType((*NodeEventMessage)(nil), "internal.NodeEventMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") - proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*CreateViewMessage)(nil), "internal.CreateViewMessage") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") + proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") + proto.RegisterType((*ResizeSource)(nil), "internal.ResizeSource") + proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") + proto.RegisterType((*SetCoordinatorMessage)(nil), "internal.SetCoordinatorMessage") + proto.RegisterType((*UpdateCoordinatorMessage)(nil), "internal.UpdateCoordinatorMessage") + proto.RegisterType((*Topology)(nil), "internal.Topology") + proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -810,18 +1224,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.ColumnLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel))) - i += copy(dAtA[i:], m.ColumnLabel) - } - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } return i, nil } @@ -840,12 +1242,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.RowLabel) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel))) - i += copy(dAtA[i:], m.RowLabel) - } if m.InverseEnabled { dAtA[i] = 0x10 i++ @@ -1055,7 +1451,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *MaxSlicesResponse) Marshal() (dAtA []byte, err error) { +func (m *MaxSlices) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1065,16 +1461,32 @@ func (m *MaxSlicesResponse) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *MaxSlicesResponse) MarshalTo(dAtA []byte) (int, error) { +func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.MaxSlices) > 0 { - for k, _ := range m.MaxSlices { + if len(m.Standard) > 0 { + for k, _ := range m.Standard { dAtA[i] = 0xa i++ - v := m.MaxSlices[k] + v := m.Standard[k] + mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(v)) + } + } + if len(m.Inverse) > 0 { + for k, _ := range m.Inverse { + dAtA[i] = 0x12 + i++ + v := m.Inverse[k] mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) dAtA[i] = 0xa @@ -1256,6 +1668,82 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Field != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Field.Size())) + n9, err := m.Field.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 + } + return i, nil +} + +func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.Field) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + return i, nil +} + func (m *Frame) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1281,11 +1769,56 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n9, err := m.Meta.MarshalTo(dAtA[i:]) + n10, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 + } + if len(m.Views) > 0 { + for _, s := range m.Views { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func (m *Schema) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Schema) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Indexes) > 0 { + for _, msg := range m.Indexes { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } } return i, nil } @@ -1311,21 +1844,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if m.Meta != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n10 - } - if m.MaxSlice != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlice)) - } if len(m.Frames) > 0 { for _, msg := range m.Frames { dAtA[i] = 0x22 @@ -1338,23 +1856,6 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.Slices) > 0 { - dAtA12 := make([]byte, len(m.Slices)*10) - var j11 int - for _, num := range m.Slices { - for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j11++ - } - dAtA12[j11] = uint8(num) - j11++ - } - dAtA[i] = 0x2a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) - } if len(m.InputDefinitions) > 0 { for _, msg := range m.InputDefinitions { dAtA[i] = 0x32 @@ -1540,11 +2041,11 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Definition.Size())) - n13, err := m.Definition.MarshalTo(dAtA[i:]) + n11, err := m.Definition.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n11 } return i, nil } @@ -1579,6 +2080,148 @@ func (m *DeleteInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *URI) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *URI) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Scheme) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) + i += copy(dAtA[i:], m.Scheme) + } + if len(m.Host) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if m.Port != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) + } + return i, nil +} + +func (m *Node) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Node) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) + i += copy(dAtA[i:], m.ID) + } + if m.URI != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) + n12, err := m.URI.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n12 + } + if m.IsCoordinator { + dAtA[i] = 0x18 + i++ + if m.IsCoordinator { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.NodeID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) + i += copy(dAtA[i:], m.NodeID) + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } + return i, nil +} + +func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Event != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) + } + if m.Node != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n13, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 + } + return i, nil +} + func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1594,35 +2237,35 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Host) > 0 { + if m.Node != nil { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n14, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n14 } - if len(m.State) > 0 { + if m.MaxSlices != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i += copy(dAtA[i:], m.State) - } - if len(m.Indexes) > 0 { - for _, msg := range m.Indexes { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) + n15, err := m.MaxSlices.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n15 } - if len(m.Scheme) > 0 { - dAtA[i] = 0x22 + if m.Schema != nil { + dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) - i += copy(dAtA[i:], m.Scheme) + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n16, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 } return i, nil } @@ -1642,39 +2285,21 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.ClusterID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i += copy(dAtA[i:], m.ClusterID) + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } if len(m.Nodes) > 0 { for _, msg := range m.Nodes { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - return i, nil -} - -func (m *FrameSchema) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *FrameSchema) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0xa + dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) @@ -1727,6 +2352,42 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.View) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + return i, nil +} + func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1763,6 +2424,284 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + } + if m.Node != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n17, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n17 + } + if m.Coordinator != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) + n18, err := m.Coordinator.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n18 + } + if len(m.Sources) > 0 { + for _, msg := range m.Sources { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Schema != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) + n19, err := m.Schema.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n19 + } + if m.ClusterStatus != nil { + dAtA[i] = 0x32 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) + n20, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n20 + } + return i, nil +} + +func (m *ResizeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Node != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n21, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n21 + } + if len(m.Index) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.View) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + if m.Slice != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + } + return i, nil +} + +func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.JobID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + } + if m.Node != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) + n22, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n22 + } + if len(m.Error) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) + i += copy(dAtA[i:], m.Error) + } + return i, nil +} + +func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.New != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n23, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n23 + } + return i, nil +} + +func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.New != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) + n24, err := m.New.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n24 + } + return i, nil +} + +func (m *Topology) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Topology) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ClusterID) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i += copy(dAtA[i:], m.ClusterID) + } + if len(m.NodeIDs) > 0 { + for _, s := range m.NodeIDs { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -1793,24 +2732,12 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l - l = len(m.ColumnLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.TimeQuantum) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } return n } func (m *FrameMeta) Size() (n int) { var l int _ = l - l = len(m.RowLabel) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.InverseEnabled { n += 2 } @@ -1904,11 +2831,19 @@ func (m *Cache) Size() (n int) { return n } -func (m *MaxSlicesResponse) Size() (n int) { +func (m *MaxSlices) Size() (n int) { var l int _ = l - if len(m.MaxSlices) > 0 { - for k, v := range m.MaxSlices { + if len(m.Standard) > 0 { + for k, v := range m.Standard { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if len(m.Inverse) > 0 { + for k, v := range m.Inverse { _ = k _ = v mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) @@ -1990,6 +2925,42 @@ func (m *DeleteFrameMessage) Size() (n int) { return n } +func (m *CreateFieldMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Field != nil { + l = m.Field.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DeleteFieldMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *Frame) Size() (n int) { var l int _ = l @@ -2001,6 +2972,24 @@ func (m *Frame) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if len(m.Views) > 0 { + for _, s := range m.Views { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *Schema) Size() (n int) { + var l int + _ = l + if len(m.Indexes) > 0 { + for _, e := range m.Indexes { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } return n } @@ -2011,26 +3000,12 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - if m.MaxSlice != 0 { - n += 1 + sovPrivate(uint64(m.MaxSlice)) - } if len(m.Frames) > 0 { for _, e := range m.Frames { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } } - if len(m.Slices) > 0 { - l = 0 - for _, e := range m.Slices { - l += sovPrivate(uint64(e)) - } - n += 1 + sovPrivate(uint64(l)) + l - } if len(m.InputDefinitions) > 0 { for _, e := range m.InputDefinitions { l = e.Size() @@ -2134,25 +3109,80 @@ func (m *DeleteInputDefinitionMessage) Size() (n int) { return n } -func (m *NodeStatus) Size() (n int) { +func (m *URI) Size() (n int) { var l int _ = l + l = len(m.Scheme) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } l = len(m.Host) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.Port != 0 { + n += 1 + sovPrivate(uint64(m.Port)) + } + return n +} + +func (m *Node) Size() (n int) { + var l int + _ = l + l = len(m.ID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.URI != nil { + l = m.URI.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.IsCoordinator { + n += 2 + } + return n +} + +func (m *NodeStateMessage) Size() (n int) { + var l int + _ = l + l = len(m.NodeID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } l = len(m.State) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Indexes) > 0 { - for _, e := range m.Indexes { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } + return n +} + +func (m *NodeEventMessage) Size() (n int) { + var l int + _ = l + if m.Event != 0 { + n += 1 + sovPrivate(uint64(m.Event)) } - l = len(m.Scheme) - if l > 0 { + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *NodeStatus) Size() (n int) { + var l int + _ = l + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.MaxSlices != nil { + l = m.MaxSlices.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Schema != nil { + l = m.Schema.Size() n += 1 + l + sovPrivate(uint64(l)) } return n @@ -2161,6 +3191,14 @@ func (m *NodeStatus) Size() (n int) { func (m *ClusterStatus) Size() (n int) { var l int _ = l + l = len(m.ClusterID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } if len(m.Nodes) > 0 { for _, e := range m.Nodes { l = e.Size() @@ -2170,18 +3208,6 @@ func (m *ClusterStatus) Size() (n int) { return n } -func (m *FrameSchema) Size() (n int) { - var l int - _ = l - if len(m.Fields) > 0 { - for _, e := range m.Fields { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - } - return n -} - func (m *Field) Size() (n int) { var l int _ = l @@ -2202,6 +3228,24 @@ func (m *Field) Size() (n int) { return n } +func (m *CreateViewMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.View) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *DeleteViewMessage) Size() (n int) { var l int _ = l @@ -2220,6 +3264,121 @@ func (m *DeleteViewMessage) Size() (n int) { return n } +func (m *ResizeInstruction) Size() (n int) { + var l int + _ = l + if m.JobID != 0 { + n += 1 + sovPrivate(uint64(m.JobID)) + } + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Coordinator != nil { + l = m.Coordinator.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.Sources) > 0 { + for _, e := range m.Sources { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.ClusterStatus != nil { + l = m.ClusterStatus.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *ResizeSource) Size() (n int) { + var l int + _ = l + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.View) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovPrivate(uint64(m.Slice)) + } + return n +} + +func (m *ResizeInstructionComplete) Size() (n int) { + var l int + _ = l + if m.JobID != 0 { + n += 1 + sovPrivate(uint64(m.JobID)) + } + if m.Node != nil { + l = m.Node.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *SetCoordinatorMessage) Size() (n int) { + var l int + _ = l + if m.New != nil { + l = m.New.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *UpdateCoordinatorMessage) Size() (n int) { + var l int + _ = l + if m.New != nil { + l = m.New.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *Topology) Size() (n int) { + var l int + _ = l + l = len(m.ClusterID) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.NodeIDs) > 0 { + for _, s := range m.NodeIDs { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *RecalculateCaches) Size() (n int) { + var l int + _ = l + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -2262,64 +3421,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ColumnLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2370,35 +3471,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RowLabel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) @@ -3108,7 +4180,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } return nil } -func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { +func (m *MaxSlices) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3131,15 +4203,15 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: MaxSlicesResponse: wiretype end group for non-group") + return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: MaxSlicesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Standard", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3203,8 +4275,8 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { } mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) iNdEx = postStringIndexmapkey - if m.MaxSlices == nil { - m.MaxSlices = make(map[string]uint64) + if m.Standard == nil { + m.Standard = make(map[string]uint64) } if iNdEx < postIndex { var valuekey uint64 @@ -3237,10 +4309,116 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { break } } - m.MaxSlices[mapkey] = mapvalue + m.Standard[mapkey] = mapvalue } else { var mapvalue uint64 - m.MaxSlices[mapkey] = mapvalue + m.Standard[mapkey] = mapvalue + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.Inverse == nil { + m.Inverse = make(map[string]uint64) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Inverse[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.Inverse[mapkey] = mapvalue } iNdEx = postIndex default: @@ -3822,6 +5000,284 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateFieldMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Field == nil { + m.Field = &Field{} + } + if err := m.Field.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteFieldMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteFieldMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Frame) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3913,6 +5369,116 @@ func (m *Frame) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Views", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Schema) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Schema: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Schema: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Indexes = append(m.Indexes, &Index{}) + if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3992,58 +5558,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &IndexMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlice", wireType) - } - m.MaxSlice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.MaxSlice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) @@ -4075,68 +5589,6 @@ func (m *Index) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 5: - if wireType == 0 { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) - } else if wireType == 2 { - var packedLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - packedLen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if packedLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + packedLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - for iNdEx < postIndex { - var v uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Slices = append(m.Slices, v) - } - } else { - return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) - } case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitions", wireType) @@ -4913,6 +6365,475 @@ func (m *DeleteInputDefinitionMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *URI) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: URI: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: URI: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Scheme", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Scheme = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + } + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Node) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Node: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Node: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field URI", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.URI == nil { + m.URI = &URI{} + } + if err := m.URI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsCoordinator", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsCoordinator = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeStateMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeStateMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeEventMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeEventMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Event", wireType) + } + m.Event = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Event |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4944,65 +6865,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Host = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.State = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5026,16 +6889,18 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Indexes = append(m.Indexes, &Index{}) - if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 4: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Scheme", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -5045,20 +6910,57 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Scheme = string(dAtA[iNdEx:postIndex]) + if m.MaxSlices == nil { + m.MaxSlices = &MaxSlices{} + } + if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -5111,6 +7013,64 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) } @@ -5136,7 +7096,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Nodes = append(m.Nodes, &NodeStatus{}) + m.Nodes = append(m.Nodes, &Node{}) if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -5162,87 +7122,6 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } -func (m *FrameSchema) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: FrameSchema: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: FrameSchema: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Fields = append(m.Fields, &Field{}) - if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5389,6 +7268,143 @@ func (m *Field) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateViewMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateViewMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5526,6 +7542,882 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeInstruction: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeInstruction: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Coordinator", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Coordinator == nil { + m.Coordinator = &Node{} + } + if err := m.Coordinator.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sources = append(m.Sources, &ResizeSource{}) + if err := m.Sources[len(m.Sources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterStatus", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClusterStatus == nil { + m.ClusterStatus = &ClusterStatus{} + } + if err := m.ClusterStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeInstructionComplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeInstructionComplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + m.JobID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.JobID |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Node == nil { + m.Node = &Node{} + } + if err := m.Node.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetCoordinatorMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetCoordinatorMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field New", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.New == nil { + m.New = &Node{} + } + if err := m.New.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UpdateCoordinatorMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UpdateCoordinatorMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field New", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.New == nil { + m.New = &Node{} + } + if err := m.New.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Topology) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Topology: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Topology: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeIDs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeIDs = append(m.NodeIDs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RecalculateCaches: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RecalculateCaches: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -5634,65 +8526,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 948 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, - 0x10, 0x65, 0x3c, 0x63, 0xaf, 0x5d, 0x26, 0x1b, 0xa7, 0x09, 0x2b, 0x6f, 0x14, 0x19, 0xab, 0x0f, - 0x6c, 0x88, 0x44, 0x0e, 0x41, 0x5a, 0x01, 0xcb, 0x01, 0x36, 0xce, 0x2a, 0x16, 0x78, 0x81, 0xf6, - 0x6a, 0xb9, 0x21, 0x75, 0x9c, 0x62, 0x77, 0x94, 0xf1, 0x8c, 0x99, 0x69, 0x27, 0x31, 0x07, 0x8e, - 0x7c, 0x03, 0x12, 0x47, 0x7e, 0x86, 0x23, 0x9f, 0x80, 0xc2, 0x85, 0x3f, 0x40, 0xe2, 0x84, 0xba, - 0xba, 0x7b, 0x66, 0x6c, 0xc7, 0x8e, 0xb2, 0xb7, 0xae, 0x57, 0xd5, 0x55, 0xaf, 0xab, 0xab, 0xaa, - 0x1b, 0x36, 0x26, 0x69, 0x78, 0x21, 0x15, 0x1e, 0x4c, 0xd2, 0x44, 0x25, 0xac, 0x1e, 0xc6, 0x0a, - 0xd3, 0x58, 0x46, 0xfc, 0x6b, 0x68, 0xf4, 0xe3, 0x33, 0xbc, 0x1a, 0xa0, 0x92, 0xac, 0x0b, 0xcd, - 0xa3, 0x24, 0x9a, 0x8e, 0xe3, 0xaf, 0xe4, 0x29, 0x46, 0x6d, 0xaf, 0xeb, 0xed, 0x35, 0x44, 0x19, - 0xd2, 0x16, 0x2f, 0xc2, 0x31, 0x7e, 0x3b, 0x95, 0xb1, 0x9a, 0x8e, 0xdb, 0x15, 0x63, 0x51, 0x82, - 0xf8, 0x7f, 0x1e, 0x34, 0x9e, 0xa5, 0x72, 0x8c, 0xe4, 0x71, 0x07, 0xea, 0x22, 0xb9, 0x2c, 0xbb, - 0xcb, 0x65, 0xf6, 0x3e, 0xdc, 0xef, 0xc7, 0x17, 0x98, 0x66, 0x78, 0x1c, 0xcb, 0xd3, 0x08, 0xcf, - 0xc8, 0x5d, 0x5d, 0x2c, 0xa0, 0x6c, 0x17, 0x1a, 0x47, 0x72, 0xf4, 0x1a, 0x5f, 0xcc, 0x26, 0xd8, - 0xf6, 0xc9, 0x49, 0x01, 0xe4, 0xda, 0x61, 0xf8, 0x13, 0xb6, 0x83, 0xae, 0xb7, 0xb7, 0x21, 0x0a, - 0x60, 0x91, 0x6f, 0x75, 0x89, 0x2f, 0xe3, 0xf0, 0xb6, 0x90, 0xf1, 0xab, 0x9c, 0x43, 0x8d, 0x38, - 0xcc, 0x61, 0xec, 0x11, 0xd4, 0x9e, 0x85, 0x18, 0x9d, 0x65, 0xed, 0x7b, 0x5d, 0x7f, 0xaf, 0x79, - 0xb8, 0x79, 0xe0, 0xf2, 0x77, 0x40, 0xb8, 0xb0, 0x6a, 0xce, 0xe1, 0x7e, 0x7f, 0x3c, 0x49, 0x52, - 0x25, 0x30, 0x9b, 0x24, 0x71, 0x86, 0xac, 0x05, 0xfe, 0x71, 0x9a, 0xda, 0xb3, 0xeb, 0x25, 0xff, - 0x19, 0x5a, 0x4f, 0xa3, 0x64, 0x74, 0xde, 0x93, 0x4a, 0x0a, 0xfc, 0x71, 0x8a, 0x99, 0x62, 0xdb, - 0x50, 0xa5, 0x5b, 0xb0, 0x76, 0x46, 0xd0, 0x28, 0x65, 0xd2, 0xa6, 0xd9, 0x08, 0x1a, 0xa5, 0xfd, - 0x94, 0x8a, 0x40, 0x18, 0x41, 0xa3, 0xc3, 0x28, 0x1c, 0x99, 0x14, 0x04, 0xc2, 0x08, 0x8c, 0x41, - 0xf0, 0x32, 0xc4, 0x4b, 0x7b, 0x6e, 0x5a, 0xf3, 0x3e, 0x6c, 0x95, 0xe2, 0x5b, 0x9a, 0x0f, 0xa0, - 0x26, 0x92, 0xcb, 0x7e, 0x2f, 0x6b, 0x7b, 0x5d, 0x7f, 0x2f, 0x10, 0x56, 0xa2, 0xec, 0xd2, 0xf5, - 0x6b, 0x55, 0x85, 0x54, 0x05, 0xc0, 0x1f, 0x42, 0x95, 0x52, 0xad, 0x4f, 0x59, 0xec, 0xd5, 0x4b, - 0xfe, 0x9b, 0x07, 0x5b, 0x03, 0x79, 0x45, 0x34, 0xb2, 0x3c, 0xcc, 0x09, 0x34, 0x72, 0x90, 0xac, - 0x9b, 0x87, 0xfb, 0x45, 0x2e, 0x97, 0xec, 0x0b, 0xe4, 0x38, 0x56, 0xe9, 0x4c, 0x14, 0x9b, 0x77, - 0x3e, 0x83, 0xfb, 0xf3, 0x4a, 0xcd, 0xe1, 0x1c, 0x67, 0x2e, 0xd3, 0xe7, 0x38, 0xd3, 0x39, 0xb9, - 0x90, 0xd1, 0xd4, 0xe4, 0x2f, 0x10, 0x46, 0xf8, 0xb4, 0xf2, 0xb1, 0xc7, 0xbf, 0x07, 0x76, 0x94, - 0xa2, 0x54, 0x48, 0x0e, 0x06, 0x98, 0x65, 0xf2, 0x15, 0xae, 0xbe, 0x05, 0x93, 0xd9, 0x4a, 0x39, - 0xb3, 0xbb, 0xd0, 0xe8, 0x67, 0xb6, 0x50, 0xe9, 0x26, 0xea, 0xa2, 0x00, 0xf8, 0x3e, 0xb0, 0x1e, - 0x46, 0xa8, 0xd0, 0xf6, 0xd6, 0x1a, 0xff, 0x7c, 0xe8, 0xb8, 0xdc, 0x6e, 0xcb, 0x1e, 0x41, 0xa0, - 0xdb, 0x8a, 0xa8, 0x34, 0x0f, 0xdf, 0x29, 0x52, 0x97, 0xf7, 0xb0, 0x20, 0x03, 0x1e, 0x3a, 0xa7, - 0xb6, 0x15, 0x6f, 0x39, 0xe0, 0x0d, 0x65, 0xe6, 0x42, 0xf9, 0x8b, 0xa1, 0xf2, 0xe6, 0xb6, 0xa1, - 0x3e, 0x77, 0x67, 0x7d, 0xd3, 0x50, 0xbc, 0x67, 0x51, 0x5d, 0xae, 0xcf, 0xb5, 0xd6, 0xec, 0xa1, - 0xf5, 0xea, 0x23, 0x2f, 0xf2, 0xf8, 0xc7, 0xb3, 0x21, 0xef, 0xe6, 0x66, 0x21, 0x73, 0x7a, 0x62, - 0xb9, 0xc2, 0xb2, 0x1d, 0x96, 0xcb, 0x34, 0x07, 0x74, 0xd4, 0xac, 0x1d, 0x2c, 0xcd, 0x01, 0x8d, - 0x0b, 0xab, 0xd6, 0xed, 0x64, 0x8b, 0xbc, 0x6a, 0xda, 0xc9, 0x48, 0xec, 0x18, 0x5a, 0xfd, 0x78, - 0x32, 0x55, 0x3d, 0xfc, 0x21, 0x8c, 0x43, 0x15, 0x26, 0x71, 0xd6, 0xae, 0x91, 0xab, 0x87, 0x65, - 0x46, 0x73, 0x16, 0x62, 0x69, 0x0b, 0xff, 0xc5, 0x83, 0xcd, 0x05, 0x70, 0xc5, 0xa1, 0x1d, 0xdf, - 0xca, 0x7a, 0xbe, 0x8f, 0xf3, 0x01, 0xe7, 0x93, 0x61, 0x67, 0x25, 0x9b, 0xf9, 0x79, 0xf7, 0xbb, - 0x07, 0xdb, 0x37, 0x19, 0xdc, 0xc8, 0xa6, 0x03, 0xf0, 0x4d, 0x1a, 0x8e, 0x65, 0x3a, 0xfb, 0x12, - 0x67, 0x76, 0xd6, 0x97, 0x10, 0xf6, 0x1d, 0x3c, 0x58, 0xf0, 0xf5, 0xc5, 0xc8, 0xa4, 0xc8, 0x90, - 0x7a, 0x6f, 0x25, 0x29, 0x63, 0x27, 0x56, 0x6c, 0xe7, 0xff, 0x7a, 0xf0, 0xee, 0x8d, 0xaa, 0xa2, - 0x1e, 0xbd, 0x72, 0xe9, 0xef, 0x43, 0xeb, 0xa5, 0x1e, 0x15, 0x3d, 0xcc, 0x54, 0x18, 0x4b, 0x6d, - 0x69, 0x0b, 0x76, 0x09, 0x67, 0x7d, 0xa8, 0x13, 0x36, 0x90, 0x13, 0x4b, 0xf3, 0xc3, 0x5b, 0x68, - 0x1e, 0x38, 0x7b, 0x33, 0xd3, 0xf2, 0xed, 0x9a, 0x0c, 0x4d, 0x5d, 0x37, 0xc2, 0x49, 0xd8, 0x79, - 0x02, 0x1b, 0x73, 0x1b, 0xee, 0x34, 0xe7, 0x12, 0xd8, 0x75, 0xb3, 0x65, 0x8e, 0xc9, 0xfa, 0x2e, - 0xfd, 0x04, 0xa0, 0x30, 0xb5, 0x03, 0x60, 0x4d, 0x7d, 0x96, 0x8c, 0xf9, 0x09, 0xec, 0xba, 0xc1, - 0x77, 0x87, 0x80, 0xae, 0x5a, 0x2a, 0x45, 0xb5, 0xf0, 0x19, 0xc0, 0xf3, 0xe4, 0x0c, 0x87, 0x4a, - 0xaa, 0x69, 0xa6, 0x2d, 0x4e, 0x92, 0x4c, 0xb9, 0x7a, 0xd2, 0x6b, 0x1a, 0xcc, 0x4a, 0xaa, 0x7c, - 0x98, 0x90, 0xc0, 0x3e, 0x80, 0x7b, 0xe4, 0x14, 0x5d, 0xd9, 0x6c, 0x2e, 0xf4, 0xba, 0x70, 0x7a, - 0xea, 0xd2, 0xd1, 0x6b, 0x1c, 0x9b, 0x47, 0xb3, 0x21, 0xac, 0xc4, 0x9f, 0xc0, 0xc6, 0x51, 0x34, - 0xcd, 0x14, 0xa6, 0x36, 0xfa, 0x3e, 0x54, 0x35, 0x17, 0xf7, 0x64, 0x6d, 0x17, 0x1e, 0x0b, 0x8a, - 0xc2, 0x98, 0xf0, 0xc7, 0xd0, 0xa4, 0x2a, 0x22, 0x5f, 0xb2, 0xf4, 0x75, 0xf0, 0xd6, 0x7f, 0x1d, - 0x86, 0x50, 0x5d, 0xdd, 0x3a, 0x0c, 0x02, 0xfa, 0xfd, 0xd8, 0x04, 0xd1, 0xc7, 0xa7, 0x05, 0xfe, - 0x20, 0x34, 0xd7, 0xe3, 0x0b, 0xbd, 0x24, 0x44, 0x5e, 0xd1, 0x61, 0x34, 0x22, 0xf5, 0xdb, 0xb2, - 0x65, 0xae, 0x43, 0xbf, 0xfc, 0x6f, 0xf2, 0x0a, 0xb8, 0x0f, 0x84, 0x5f, 0x7c, 0x20, 0x9e, 0xb6, - 0xfe, 0xb8, 0xee, 0x78, 0x7f, 0x5e, 0x77, 0xbc, 0xbf, 0xae, 0x3b, 0xde, 0xaf, 0x7f, 0x77, 0xde, - 0x3a, 0xad, 0xd1, 0xaf, 0xf2, 0xa3, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x47, 0xdd, 0xdd, 0x8e, - 0x66, 0x0a, 0x00, 0x00, + // 1307 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xff, 0xaf, 0xd7, 0x76, 0xe2, 0xc7, 0x71, 0xe2, 0x4c, 0xd3, 0xfc, 0x37, 0x55, 0xe4, 0x9a, + 0x51, 0xa1, 0xa6, 0x12, 0x51, 0x49, 0x25, 0x44, 0x03, 0x95, 0x4a, 0x62, 0x57, 0x5d, 0x20, 0x51, + 0x19, 0x27, 0x41, 0x42, 0x02, 0x69, 0x62, 0x0f, 0xe9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2, + 0x1e, 0x38, 0x22, 0x24, 0xc4, 0x1d, 0x71, 0xe5, 0xcb, 0x70, 0xe4, 0x13, 0x20, 0x14, 0x3e, 0x04, + 0x47, 0xd0, 0xbc, 0xed, 0xae, 0xdf, 0x92, 0x26, 0x70, 0xdb, 0xe7, 0xfd, 0x37, 0xcf, 0xdb, 0xcc, + 0x42, 0xa5, 0x1f, 0x79, 0xa7, 0x94, 0xb3, 0x8d, 0x7e, 0x14, 0xf2, 0x10, 0xcd, 0x7b, 0x01, 0x67, + 0x51, 0x40, 0x7d, 0x5c, 0x86, 0x92, 0x1b, 0x74, 0xd9, 0xf9, 0x2e, 0xe3, 0x14, 0xff, 0x6e, 0x41, + 0xe9, 0x59, 0x44, 0x7b, 0x4c, 0x50, 0xe8, 0x2d, 0x58, 0x74, 0x83, 0x53, 0x16, 0xc5, 0xac, 0x15, + 0xd0, 0x23, 0x9f, 0x75, 0x9d, 0x5c, 0xdd, 0x6a, 0xcc, 0x93, 0x31, 0x2e, 0x5a, 0x87, 0xd2, 0x0e, + 0xed, 0xbc, 0x64, 0xfb, 0xc3, 0x3e, 0x73, 0xec, 0xba, 0xd5, 0x28, 0x91, 0x94, 0x91, 0x48, 0xdb, + 0xde, 0x2b, 0xe6, 0xe4, 0xeb, 0x56, 0xa3, 0x42, 0x52, 0x06, 0xaa, 0x43, 0x79, 0xdf, 0xeb, 0xb1, + 0xcf, 0x06, 0x34, 0xe0, 0x83, 0x9e, 0x53, 0x90, 0xd6, 0x59, 0x16, 0xc2, 0xb0, 0x40, 0x68, 0x70, + 0x9c, 0x60, 0x28, 0x4a, 0x0c, 0x23, 0x3c, 0x74, 0x1f, 0x8a, 0xcf, 0x3c, 0xe6, 0x77, 0x63, 0x67, + 0xae, 0x6e, 0x37, 0xca, 0x9b, 0x4b, 0x1b, 0xe6, 0x7c, 0x1b, 0x92, 0x4f, 0xb4, 0x18, 0x63, 0x58, + 0x74, 0x7b, 0xfd, 0x30, 0xe2, 0x84, 0xc5, 0xfd, 0x30, 0x88, 0x19, 0xaa, 0x82, 0xdd, 0x8a, 0x22, + 0xc7, 0x92, 0x81, 0xc5, 0x27, 0xfe, 0x16, 0xaa, 0xdb, 0x7e, 0xd8, 0x39, 0x69, 0x52, 0x4e, 0x09, + 0xfb, 0x66, 0xc0, 0x62, 0x8e, 0x56, 0xa0, 0x20, 0xb3, 0xa4, 0xf5, 0x14, 0x21, 0xb8, 0x32, 0x5b, + 0x32, 0x2f, 0x25, 0xa2, 0x08, 0xc1, 0x95, 0xf6, 0x32, 0x15, 0x79, 0xa2, 0x08, 0xc1, 0x6d, 0xfb, + 0x5e, 0x47, 0xa5, 0x20, 0x4f, 0x14, 0x81, 0x10, 0xe4, 0x0f, 0x3d, 0x76, 0xa6, 0xcf, 0x2d, 0xbf, + 0xb1, 0x0b, 0xcb, 0x99, 0xf8, 0x1a, 0xe6, 0x2a, 0x14, 0x49, 0x78, 0xe6, 0x36, 0x63, 0xc7, 0xaa, + 0xdb, 0x8d, 0x3c, 0xd1, 0x94, 0xcc, 0x6e, 0xe8, 0x0f, 0x7a, 0x81, 0x10, 0xe5, 0xa4, 0x28, 0x65, + 0xe0, 0x35, 0x28, 0xc8, 0x54, 0x8b, 0x53, 0xa6, 0xb6, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0x69, 0x97, + 0x9e, 0x4b, 0x18, 0x31, 0x7a, 0x02, 0xf3, 0x6d, 0x4e, 0x83, 0x2e, 0x8d, 0xba, 0x52, 0xa9, 0xbc, + 0xf9, 0x46, 0x9a, 0xc2, 0x44, 0x6d, 0xc3, 0xe8, 0xb4, 0x02, 0x1e, 0x0d, 0x49, 0x62, 0x82, 0xb6, + 0x60, 0x4e, 0xf7, 0x84, 0xc4, 0x50, 0xde, 0xac, 0x4f, 0xb3, 0x4e, 0xda, 0x46, 0x18, 0x1b, 0x83, + 0x3b, 0x1f, 0x40, 0x65, 0xc4, 0xad, 0xc0, 0x7a, 0xc2, 0x86, 0xa6, 0x22, 0x27, 0x6c, 0x28, 0x72, + 0x77, 0x4a, 0xfd, 0x81, 0xca, 0x73, 0x9e, 0x28, 0x62, 0x2b, 0xf7, 0xbe, 0x75, 0x67, 0x0b, 0x16, + 0xb2, 0x5e, 0xaf, 0x63, 0x8b, 0xbf, 0x02, 0xb4, 0x13, 0x31, 0xca, 0x99, 0x84, 0xb7, 0xcb, 0xe2, + 0x98, 0x1e, 0xb3, 0xd9, 0x95, 0x56, 0xd5, 0xcb, 0x65, 0xab, 0xb7, 0x0e, 0x25, 0x37, 0x36, 0x07, + 0xb7, 0x65, 0x5f, 0xa6, 0x0c, 0xfc, 0x00, 0x50, 0x93, 0xf9, 0x8c, 0x33, 0x3d, 0x5f, 0x97, 0xf8, + 0xc7, 0x6d, 0x83, 0xe5, 0x6a, 0x5d, 0x74, 0x1f, 0xf2, 0x62, 0x3c, 0x25, 0x94, 0xf2, 0xe6, 0xad, + 0x34, 0xd3, 0xc9, 0x1c, 0x13, 0xa9, 0x80, 0x3d, 0xe3, 0x54, 0x8f, 0xf4, 0x15, 0x07, 0x9c, 0xd2, + 0xca, 0x26, 0x94, 0x3d, 0x1e, 0x2a, 0x59, 0x12, 0x3a, 0xd4, 0x53, 0x73, 0xd6, 0x9b, 0x86, 0xc2, + 0xc7, 0x09, 0x58, 0x31, 0xa9, 0x37, 0x01, 0xfb, 0x26, 0x14, 0xa4, 0xad, 0x46, 0x3b, 0xb1, 0x03, + 0x94, 0x14, 0x1f, 0x26, 0x50, 0x6f, 0x1a, 0x68, 0x25, 0x1b, 0xa8, 0x64, 0xfc, 0x7e, 0xa1, 0x75, + 0xc5, 0x4c, 0xef, 0x09, 0x1b, 0xe5, 0x49, 0x7e, 0xcf, 0xae, 0xd9, 0x58, 0x22, 0x85, 0x6f, 0xb1, + 0x04, 0x62, 0xc7, 0xae, 0xdb, 0xc2, 0xb7, 0x24, 0xf0, 0x23, 0x28, 0xb6, 0x3b, 0x2f, 0x59, 0x8f, + 0xa2, 0xb7, 0xc5, 0xa4, 0x75, 0xd9, 0x39, 0x8b, 0xf5, 0x9c, 0x2e, 0x8d, 0xd5, 0x9f, 0x18, 0x39, + 0xfe, 0xc1, 0xd2, 0x67, 0x9a, 0x81, 0xa8, 0x28, 0x63, 0xc7, 0x4e, 0x7e, 0x62, 0x65, 0x0a, 0x3e, + 0xd1, 0x62, 0xd4, 0x82, 0xaa, 0x1b, 0xf4, 0x07, 0xbc, 0xc9, 0xbe, 0xf6, 0x02, 0x8f, 0x7b, 0x61, + 0x10, 0x3b, 0x45, 0x69, 0xb2, 0x96, 0x0d, 0x3d, 0xa2, 0x41, 0x26, 0x4c, 0xf0, 0x77, 0x16, 0x2c, + 0x8d, 0x31, 0xaf, 0xc0, 0x95, 0xbb, 0x1c, 0xd7, 0x7b, 0xc9, 0xce, 0xb7, 0xa5, 0x62, 0x6d, 0x26, + 0x9a, 0xd1, 0x2b, 0xe0, 0x17, 0x0b, 0x56, 0xa6, 0x29, 0x4c, 0x45, 0x53, 0x03, 0x78, 0x11, 0x79, + 0x3d, 0x1a, 0x0d, 0x3f, 0x61, 0x43, 0x7d, 0xfd, 0x65, 0x38, 0xe8, 0x73, 0x58, 0x1d, 0xf3, 0xf5, + 0x51, 0x47, 0xa5, 0x48, 0x81, 0xba, 0x3b, 0x13, 0x94, 0xd2, 0x23, 0x33, 0xcc, 0xf1, 0x5f, 0x16, + 0xdc, 0x9e, 0x2a, 0x4a, 0x7b, 0xd2, 0xca, 0xf6, 0xe4, 0x03, 0xa8, 0x1e, 0x8a, 0xcd, 0xd6, 0x64, + 0x31, 0xf7, 0x02, 0x2a, 0x34, 0x75, 0xd3, 0x4e, 0xf0, 0x91, 0x0b, 0xf3, 0x92, 0xb7, 0x4b, 0xfb, + 0x1a, 0xe6, 0x3b, 0x57, 0xc0, 0xdc, 0x30, 0xfa, 0x7a, 0xf1, 0x1b, 0x52, 0x80, 0x91, 0x17, 0x91, + 0xb9, 0xd5, 0x24, 0x21, 0x56, 0xfa, 0x88, 0xc1, 0xb5, 0xd6, 0x72, 0x08, 0xeb, 0x66, 0x15, 0x8e, + 0x20, 0xb9, 0x7c, 0x52, 0x1f, 0x03, 0xa4, 0xaa, 0x7a, 0x03, 0x5c, 0xd2, 0x9f, 0x19, 0x65, 0xfc, + 0x1c, 0xd6, 0xcd, 0x9e, 0xbe, 0x46, 0x40, 0xd3, 0x2d, 0xb9, 0xb4, 0x5b, 0x70, 0x0b, 0xec, 0x03, + 0xe2, 0x8a, 0xbb, 0x5a, 0x4e, 0xab, 0x29, 0x91, 0xa6, 0x84, 0xc9, 0xf3, 0x30, 0xe6, 0xc6, 0x44, + 0x7c, 0x0b, 0xde, 0x8b, 0x30, 0xe2, 0x12, 0x71, 0x85, 0xc8, 0x6f, 0xfc, 0x25, 0xe4, 0xf7, 0xc2, + 0x2e, 0x43, 0x8b, 0x90, 0x73, 0x9b, 0xda, 0x47, 0xce, 0x6d, 0xa2, 0xbb, 0xd2, 0xbd, 0xde, 0x21, + 0x95, 0xf4, 0x70, 0x07, 0xc4, 0x25, 0x32, 0xf0, 0x3d, 0xa8, 0xb8, 0xf1, 0x4e, 0x18, 0x46, 0x5d, + 0x51, 0xea, 0x30, 0xd2, 0x77, 0xd2, 0x28, 0x13, 0x3f, 0x85, 0xaa, 0x70, 0xdf, 0xe6, 0x94, 0x27, + 0x9b, 0x7a, 0x15, 0x8a, 0x82, 0x97, 0x84, 0xd3, 0x94, 0xbc, 0xf7, 0x84, 0x9e, 0x59, 0x80, 0x92, + 0xc0, 0x9f, 0x2a, 0x0f, 0xad, 0x53, 0x16, 0xf0, 0x4c, 0x96, 0x24, 0x2d, 0x1d, 0x54, 0x88, 0x22, + 0x10, 0x56, 0x47, 0xd1, 0x98, 0x17, 0x53, 0xcc, 0x82, 0x4b, 0xa4, 0x0c, 0xff, 0x68, 0x01, 0x18, + 0x40, 0x83, 0x38, 0x31, 0xb1, 0x66, 0x9b, 0xa0, 0x77, 0x33, 0x6f, 0x97, 0xc9, 0x9d, 0x9a, 0x88, + 0x48, 0xe6, 0x85, 0xd3, 0x30, 0x2b, 0x54, 0x37, 0x47, 0x35, 0xd5, 0x57, 0x7c, 0x5d, 0x26, 0x71, + 0x6d, 0x56, 0x76, 0xfc, 0x41, 0xcc, 0x59, 0xa4, 0x11, 0x89, 0x37, 0x96, 0x62, 0x24, 0xf9, 0x49, + 0x19, 0xd3, 0x53, 0x84, 0xee, 0x41, 0x41, 0x20, 0x35, 0x7b, 0x60, 0xfc, 0x18, 0x4a, 0x88, 0xdb, + 0xfa, 0x26, 0x99, 0xba, 0x7b, 0x10, 0xe4, 0xe5, 0x8b, 0x5a, 0xb7, 0x8b, 0x7c, 0x4c, 0x57, 0xc1, + 0xde, 0xf5, 0x54, 0x7f, 0xdb, 0x44, 0x7c, 0x4a, 0x0e, 0x3d, 0x97, 0xf3, 0x27, 0x38, 0x54, 0xbc, + 0x25, 0x96, 0xd5, 0x00, 0x89, 0xbb, 0xe3, 0x26, 0xf7, 0x9b, 0x79, 0x94, 0xda, 0x99, 0x47, 0x69, + 0x1b, 0x96, 0xd5, 0x90, 0xfc, 0x97, 0x4e, 0x7f, 0xce, 0xc1, 0x32, 0x61, 0xb1, 0xf7, 0x8a, 0xb9, + 0x41, 0xcc, 0xa3, 0x41, 0xb2, 0xe0, 0x3e, 0x0e, 0x8f, 0x74, 0xaa, 0x6d, 0xa2, 0x88, 0xd7, 0xe9, + 0x24, 0xf4, 0x10, 0xca, 0xe3, 0xdd, 0x3f, 0xa9, 0x9a, 0x55, 0x41, 0x0f, 0x61, 0xae, 0x1d, 0x0e, + 0xa2, 0x4e, 0x72, 0x0d, 0xae, 0xa6, 0xda, 0x0a, 0x99, 0x12, 0x13, 0xa3, 0x96, 0xe9, 0xa3, 0xc2, + 0xe5, 0x7d, 0x84, 0x9e, 0x8c, 0xf5, 0x91, 0xfc, 0x73, 0x29, 0x6f, 0xfe, 0x3f, 0x35, 0x18, 0x11, + 0x93, 0x51, 0x6d, 0xfc, 0xbd, 0x05, 0x0b, 0x59, 0x08, 0xaf, 0x35, 0x18, 0x49, 0x45, 0x72, 0x53, + 0x2b, 0x62, 0x4f, 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0xce, 0x2d, 0x64, 0xde, 0xb9, 0xf8, 0x04, + 0xd6, 0x26, 0xca, 0xb4, 0x13, 0xf6, 0xfa, 0xa2, 0x1f, 0xfe, 0x45, 0xb9, 0xc4, 0xca, 0x88, 0x22, + 0x5d, 0xa8, 0x12, 0x51, 0x04, 0x7e, 0x0c, 0xb7, 0xdb, 0x8c, 0x67, 0x8a, 0x64, 0xba, 0xad, 0x0e, + 0xf6, 0x1e, 0x3b, 0x9b, 0x71, 0x7c, 0x21, 0xc2, 0x1f, 0x82, 0x73, 0xd0, 0xef, 0x52, 0xce, 0x6e, + 0x64, 0xbd, 0x0d, 0xf3, 0xfb, 0x61, 0x3f, 0xf4, 0xc3, 0xe3, 0xe1, 0x15, 0x23, 0xef, 0xc0, 0x9c, + 0xda, 0x8f, 0xea, 0x91, 0x52, 0x22, 0x86, 0xc4, 0xb7, 0x44, 0x43, 0x77, 0xa8, 0xdf, 0x19, 0xf8, + 0x02, 0x86, 0xf8, 0xf7, 0x8a, 0xb7, 0xab, 0xbf, 0x5e, 0xd4, 0xac, 0xdf, 0x2e, 0x6a, 0xd6, 0x1f, + 0x17, 0x35, 0xeb, 0xa7, 0x3f, 0x6b, 0xff, 0x3b, 0x2a, 0xca, 0xbf, 0xf0, 0x47, 0xff, 0x04, 0x00, + 0x00, 0xff, 0xff, 0xc3, 0xb3, 0xdc, 0xe3, 0x96, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 083316ddd..bd3e52b26 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,12 +3,9 @@ syntax = "proto3"; package internal; message IndexMeta { - string ColumnLabel = 1; - string TimeQuantum = 2; } message FrameMeta { - string RowLabel = 1; bool InverseEnabled = 2; string CacheType = 3; uint32 CacheSize = 4; @@ -38,8 +35,9 @@ message Cache { repeated uint64 IDs = 1; } -message MaxSlicesResponse { - map MaxSlices = 1; +message MaxSlices { + map Standard = 1; + map Inverse = 2; } message CreateSliceMessage { @@ -68,17 +66,31 @@ message DeleteFrameMessage { string Frame = 2; } +message CreateFieldMessage { + string Index = 1; + string Frame = 2; + Field Field = 3; +} + +message DeleteFieldMessage { + string Index = 1; + string Frame = 2; + string Field = 3; +} + message Frame { string Name = 1; FrameMeta Meta = 2; + repeated string Views = 3; +} + +message Schema { + repeated Index Indexes = 1; } message Index { string Name = 1; - IndexMeta Meta = 2; - uint64 MaxSlice = 3; repeated Frame Frames = 4; - repeated uint64 Slices = 5; repeated InputDefinition InputDefinitions = 6; } @@ -112,19 +124,38 @@ message DeleteInputDefinitionMessage { string Name = 2; } -message NodeStatus { - string Host = 1; +message URI { + string Scheme = 1; + string Host = 2; + uint32 Port = 3; +} + +message Node { + string ID = 1; + URI URI = 2; + bool IsCoordinator = 3; +} + +message NodeStateMessage { + string NodeID = 1; string State = 2; - repeated Index Indexes = 3; - string Scheme = 4; +} + +message NodeEventMessage { + uint32 Event = 1; + Node Node = 2; +} + +message NodeStatus { + Node Node = 1; + MaxSlices MaxSlices = 2; + Schema Schema = 3; } message ClusterStatus { - repeated NodeStatus Nodes = 1; -} - -message FrameSchema { - repeated Field Fields = 1; + string ClusterID = 1; + string State = 2; + repeated Node Nodes = 3; } message Field { @@ -134,8 +165,52 @@ message Field { int64 Max = 4; } +message CreateViewMessage { + string Index = 1; + string Frame = 2; + string View = 3; +} + message DeleteViewMessage { string Index = 1; string Frame = 2; string View = 3; } + +message ResizeInstruction { + int64 JobID = 1; + Node Node = 2; + Node Coordinator = 3; + repeated ResizeSource Sources = 4; + Schema Schema = 5; + ClusterStatus ClusterStatus = 6; +} + +message ResizeSource { + Node Node = 1; + string Index = 2; + string Frame = 3; + string View = 4; + uint64 Slice = 5; +} + +message ResizeInstructionComplete { + int64 JobID = 1; + Node Node = 2; + string Error = 3; +} + +message SetCoordinatorMessage { + Node New = 1; +} + +message UpdateCoordinatorMessage { + Node New = 1; +} + +message Topology { + string ClusterID = 1; + repeated string NodeIDs = 2; +} + +message RecalculateCaches {} diff --git a/internal/public.pb.go b/internal/public.pb.go index dfb9a819e..31447b9f0 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ It has these top-level messages: Bitmap Pair - SumCount + ValCount Bit ColumnAttrSet Attr @@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 { return 0 } -type SumCount struct { - Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` +type ValCount struct { + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *SumCount) Reset() { *m = SumCount{} } -func (m *SumCount) String() string { return proto.CompactTextString(m) } -func (*SumCount) ProtoMessage() {} -func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -func (m *SumCount) GetSum() int64 { +func (m *ValCount) GetVal() int64 { if m != nil { - return m.Sum + return m.Val } return 0 } -func (m *SumCount) GetCount() int64 { +func (m *ValCount) GetCount() int64 { if m != nil { return m.Count } @@ -358,7 +358,7 @@ type QueryResult struct { Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } @@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetSumCount() *SumCount { +func (m *QueryResult) GetValCount() *ValCount { if m != nil { - return m.SumCount + return m.ValCount } return nil } @@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") - proto.RegisterType((*SumCount)(nil), "internal.SumCount") + proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") @@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *SumCount) Marshal() (dAtA []byte, err error) { +func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *SumCount) MarshalTo(dAtA []byte) (int, error) { +func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Sum != 0 { + if m.Val != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) } if m.Count != 0 { dAtA[i] = 0x10 @@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.SumCount != nil { + if m.ValCount != nil { dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size())) - n6, err := m.SumCount.MarshalTo(dAtA[i:]) + i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) + n6, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) { return n } -func (m *SumCount) Size() (n int) { +func (m *ValCount) Size() (n int) { var l int _ = l - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Val != 0 { + n += 1 + sovPublic(uint64(m.Val)) } if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) @@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) { if m.Changed { n += 2 } - if m.SumCount != nil { - l = m.SumCount.Size() + if m.ValCount != nil { + l = m.ValCount.Size() n += 1 + l + sovPublic(uint64(l)) } if m.Type != 0 { @@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } -func (m *SumCount) Unmarshal(dAtA []byte) error { +func (m *ValCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SumCount: wiretype end group for non-group") + return fmt.Errorf("proto: ValCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType) } - m.Sum = 0 + m.Val = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= (int64(b) & 0x7F) << shift + m.Val |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { m.Changed = bool(v != 0) case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.SumCount == nil { - m.SumCount = &SumCount{} + if m.ValCount == nil { + m.ValCount = &ValCount{} } - if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{ // 705 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, - 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3, + 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3, 0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76, 0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce, 0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6, - 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c, - 0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85, - 0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45, - 0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a, - 0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7, - 0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7, - 0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a, - 0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9, - 0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb, - 0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21, - 0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, - 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94, - 0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, + 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce, + 0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42, + 0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22, + 0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15, + 0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb, + 0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b, + 0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45, + 0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec, + 0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed, + 0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10, + 0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, + 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94, + 0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, 0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb, 0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80, 0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd, 0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76, - 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36, - 0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc, - 0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b, - 0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4, - 0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a, - 0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad, - 0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d, - 0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2, - 0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5, - 0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23, - 0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8, - 0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b, - 0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03, - 0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad, - 0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8, - 0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f, - 0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91, - 0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc, - 0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f, - 0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4, - 0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00, + 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d, + 0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8, + 0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7, + 0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69, + 0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55, + 0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b, + 0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b, + 0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85, + 0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a, + 0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47, + 0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0, + 0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56, + 0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06, + 0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b, + 0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0, + 0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f, + 0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23, + 0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9, + 0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde, + 0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8, + 0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 5fa40cfab..026a3748b 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,8 +14,8 @@ message Pair { uint64 Count = 2; } -message SumCount { - int64 Sum = 1; +message ValCount { + int64 Val = 1; int64 Count = 2; } @@ -64,7 +64,7 @@ message QueryResult { Bitmap Bitmap = 1; uint64 N = 2; repeated Pair Pairs = 3; - SumCount SumCount = 5; + ValCount ValCount = 5; bool Changed = 4; } diff --git a/logger.go b/logger.go new file mode 100644 index 000000000..93b12c3f8 --- /dev/null +++ b/logger.go @@ -0,0 +1,88 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io" + "log" +) + +// Ensure nopLogger implements interface. +var _ Logger = &nopLogger{} + +// Logger represents an interface for a shared logger. +type Logger interface { + Printf(format string, v ...interface{}) + Debugf(format string, v ...interface{}) +} + +func init() { + NopLogger = &nopLogger{} +} + +// NopLogger represents a Logger that doesn't do anything. +var NopLogger Logger + +type nopLogger struct{} + +// Printf is a no-op implementation of the Logger Printf method. +func (n *nopLogger) Printf(format string, v ...interface{}) {} + +// Debugf is a no-op implementation of the Logger Debugf method. +func (n *nopLogger) Debugf(format string, v ...interface{}) {} + +// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. +type StandardLogger struct { + logger *log.Logger +} + +func NewStandardLogger(w io.Writer) *StandardLogger { + return &StandardLogger{ + logger: log.New(w, "", log.LstdFlags), + } +} + +func (s *StandardLogger) Printf(format string, v ...interface{}) { + s.logger.Printf(format, v...) +} + +func (s *StandardLogger) Debugf(format string, v ...interface{}) {} + +func (s *StandardLogger) Logger() *log.Logger { + return s.logger +} + +// VerboseLogger is an implementation of pilosa.Logger which includes debug messages. +type VerboseLogger struct { + logger *log.Logger +} + +func NewVerboseLogger(w io.Writer) *VerboseLogger { + return &VerboseLogger{ + logger: log.New(w, "", log.LstdFlags), + } +} + +func (vb *VerboseLogger) Printf(format string, v ...interface{}) { + vb.logger.Printf(format, v...) +} + +func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { + vb.logger.Printf(format, v...) +} + +func (vb *VerboseLogger) Logger() *log.Logger { + return vb.logger +} diff --git a/lru/lru.go b/lru/lru.go new file mode 100644 index 000000000..532cc45e6 --- /dev/null +++ b/lru/lru.go @@ -0,0 +1,133 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package lru implements an LRU cache. +package lru + +import "container/list" + +// Cache is an LRU cache. It is not safe for concurrent access. +type Cache struct { + // MaxEntries is the maximum number of cache entries before + // an item is evicted. Zero means no limit. + MaxEntries int + + // OnEvicted optionally specificies a callback function to be + // executed when an entry is purged from the cache. + OnEvicted func(key Key, value interface{}) + + ll *list.List + cache map[interface{}]*list.Element +} + +// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators +type Key interface{} + +type entry struct { + key Key + value interface{} +} + +// New creates a new Cache. +// If maxEntries is zero, the cache has no limit and it's assumed +// that eviction is done by the caller. +func New(maxEntries int) *Cache { + return &Cache{ + MaxEntries: maxEntries, + ll: list.New(), + cache: make(map[interface{}]*list.Element), + } +} + +// Add adds a value to the cache. +func (c *Cache) Add(key Key, value interface{}) { + if c.cache == nil { + c.cache = make(map[interface{}]*list.Element) + c.ll = list.New() + } + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value.(*entry).value = value + return + } + ele := c.ll.PushFront(&entry{key, value}) + c.cache[key] = ele + if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + c.RemoveOldest() + } +} + +// Get looks up a key's value from the cache. +func (c *Cache) Get(key Key) (value interface{}, ok bool) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.ll.MoveToFront(ele) + return ele.Value.(*entry).value, true + } + return +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key Key) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.removeElement(ele) + } +} + +// RemoveOldest removes the oldest item from the cache. +func (c *Cache) RemoveOldest() { + if c.cache == nil { + return + } + ele := c.ll.Back() + if ele != nil { + c.removeElement(ele) + } +} + +func (c *Cache) removeElement(e *list.Element) { + c.ll.Remove(e) + kv := e.Value.(*entry) + delete(c.cache, kv.key) + if c.OnEvicted != nil { + c.OnEvicted(kv.key, kv.value) + } +} + +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + if c.cache == nil { + return 0 + } + return c.ll.Len() +} + +// Clear purges all stored items from the cache. +func (c *Cache) Clear() { + if c.OnEvicted != nil { + for _, e := range c.cache { + kv := e.Value.(*entry) + c.OnEvicted(kv.key, kv.value) + } + } + c.ll = nil + c.cache = nil +} diff --git a/pilosa.go b/pilosa.go index aa1378a36..62eb88113 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,31 +36,26 @@ var ( ErrFrameExists = errors.New("frame already exists") ErrFrameNotFound = errors.New("frame not found") ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey") - ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") ErrInputDefinitionNameRequired = errors.New("input-definition name required") ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrInputDefinitionValueMap = errors.New("valueMap required for map") ErrInputDefinitionActionRequired = errors.New("field definitions require an action") ErrInputDefinitionNotFound = errors.New("input-definition not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrFieldNotFound = errors.New("field not found") + ErrFieldExists = errors.New("field already exists") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInvalidFieldValueType = errors.New("invalid field value type") + ErrFieldValueTooLow = errors.New("field value too low") + ErrFieldValueTooHigh = errors.New("field value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") @@ -73,16 +68,29 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") - ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") + ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") + ErrNodeNotCoordinator = errors.New("node is not the coordinator") + ErrResizeNotRunning = errors.New("no resize job currently running") ) +// ApiMethodNotAllowedError wraps an error value indicating that a particular +// API method is not allowed in the current cluster state. +type ApiMethodNotAllowedError struct { + error +} + +// BadRequestError wraps an error value to signify that a request could not be +// read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest +// would be returned. +type BadRequestError struct { + error +} + // Regular expression to validate index and frame names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) -// Regular expression to validate row and column labels. -var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`) - // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { @@ -144,14 +152,6 @@ func ValidateName(name string) error { return nil } -// ValidateLabel ensures that the label is a valid format. -func ValidateLabel(label string) error { - if labelRegexp.Match([]byte(label)) == false { - return ErrLabel - } - return nil -} - // StringInSlice checks for substring a in the slice. func StringInSlice(a string, list []string) bool { for _, b := range list { @@ -162,6 +162,50 @@ func StringInSlice(a string, list []string) bool { return false } +// StringSlicesAreEqual determines if two string slices are equal. +func StringSlicesAreEqual(a, b []string) bool { + + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +// SliceDiff returns the difference between two uint64 slices. +func SliceDiff(a, b []uint64) []uint64 { + m := make(map[uint64]uint64) + + for _, y := range b { + m[y]++ + } + + var ret []uint64 + for _, x := range a { + if m[x] > 0 { + m[x]-- + continue + } + ret = append(ret, x) + } + + return ret +} + // ContainsSubstring checks to see if substring a is contained in any string in the slice. func ContainsSubstring(a string, list []string) bool { for _, b := range list { diff --git a/pilosa_test.go b/pilosa_test.go index d82e27fa3..41b0d7098 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -46,30 +46,6 @@ func TestValidateNameInvalid(t *testing.T) { } } -func TestValidateLabel(t *testing.T) { - labels := []string{ - "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) != nil { - t.Fatalf("Should be valid label: %s", label) - } - } -} - -func TestValidateLabelInvalid(t *testing.T) { - labels := []string{ - "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, label := range labels { - if pilosa.ValidateLabel(label) == nil { - t.Fatalf("Should be invalid label: %s", label) - } - } -} - func TestStringInSlice(t *testing.T) { list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} substr := "localhost:10101" diff --git a/roaring/assembly.go b/roaring/assembly.go deleted file mode 100644 index ea67cf66e..000000000 --- a/roaring/assembly.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package roaring - - -// bit population count, take from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcntGo(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} - -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcntGo(x) - } - return cnt -} - -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] &^ m[i]) - } - return cnt -} - -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] & m[i]) - } - return cnt -} - -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] | m[i]) - } - return cnt -} - -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcntGo(s[i] ^ m[i]) - } - return cnt -} diff --git a/roaring/assembly_amd64.s b/roaring/assembly_amd64.s deleted file mode 100644 index 5b65e0635..000000000 --- a/roaring/assembly_amd64.s +++ /dev/null @@ -1,122 +0,0 @@ -#include "textflag.h" - -TEXT ·hasAsm(SB),4,$0-1 - MOVQ $1, AX - CPUID - SHRQ $23, CX - ANDQ $1, CX - MOVB CX, ret+0(FP) - RET - -TEXT ·POPCNTQ(SB),NOSPLIT,$0-16 - MOVQ memory+0(FP), BP - POPCNTQ BP, BX - MOVQ BX, ret+8(FP) - RET - -TEXT ·BSFQ(SB),NOSPLIT,$0-16 - MOVQ memory+0(FP), BP - BSFQ BP, BX - MOVQ BX, ret+8(FP) - RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s_base+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET - - -TEXT ·popcntAsm(SB),4,$0-16 - MOVQ x+0(FP), DX - POPCNTQ_DX_DX - MOVQ DX, ret+8(FP) - RET diff --git a/roaring/assembly_asm.go b/roaring/assembly_asm.go deleted file mode 100644 index 3eb182cce..000000000 --- a/roaring/assembly_asm.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package roaring - -func hasAsm() bool - -func BSFQ(memory uint64) int - -func POPCNTQ(memory uint64) int - - -//go:noescape -var useAsm = hasAsm() - -//go:noescape -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntXorSliceAsm(s, m []uint64) uint64 - -//go:noescape -func popcntAsm(x uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} - -func popcnt(x uint64) uint64 { - if useAsm { - return popcntAsm(x) - } - return popcntGo(x) -} diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go deleted file mode 100644 index b3e75c32c..000000000 --- a/roaring/assembly_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package roaring - -import "testing" - -func TestBSFQ(t *testing.T) { - result := BSFQ(2) - if result != 1 { - t.Fatalf("BSF INCORRECT: %d", result) - } -} - -func TestBSFQ_CompareGo(t *testing.T) { - v := uint64(1) - for i := 0; i < 64; i++ { - if BSFQ(v) != trailingZeroN(v) { - t.Fatalf("BSF INCORRECT: %d %d", BSFQ(v), trailingZeroN(v)) - } - if v == 0 { - v = 1 - } else { - v *= 2 - } - } - /* - if bsfq(0) != trailingZeroN(0) { - fmt.Println(bsfq(0)) - t.Fatalf("BSF INCORRECT") - } - */ -} -func BenchmarkBSF(b *testing.B) { - for i := 0; i < b.N; i++ { - BSFQ(uint64(i)) - } -} - -func BenchmarkTrailingZeroN(b *testing.B) { - for i := 0; i < b.N; i++ { - trailingZeroN(uint64(i)) - } -} - -func BenchmarkPOPCNTQ(b *testing.B) { - for i := 0; i < b.N; i++ { - POPCNTQ(uint64(i)) - } -} - -// This value prevents the benchmarks from being optimized out -var Result uint64 - -func BenchmarkPopcount(b *testing.B) { - for i := 0; i < b.N; i++ { - Result = popcount(uint64(i)) - } -} - -func BenchmarkPopcntAsm(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - popcntAsm(0xdeadbeef) - } -} - -func BenchmarkPopcntGo(b *testing.B) { - // run the Fib function b.N times - for n := 0; n < b.N; n++ { - Result = popcntGo(uint64(n)) - } -} - -func getData() []uint64 { - return []uint64{ - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - 0xdeadbeef, - } -} - -func BenchmarkPopcntSliceGo(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSliceGo(d) - } -} - -func BenchmarkPopcntSliceAsm(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSliceAsm(d) - } -} - -func BenchmarkPopcntSlice(b *testing.B) { - d := getData() - for n := 0; n < b.N; n++ { - popcntSlice(d) - } -} diff --git a/roaring/roaring.go b/roaring/roaring.go index 8ac2b4d4e..f25b3faca 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -22,6 +22,7 @@ import ( "hash/fnv" "io" "reflect" + "math/bits" "sort" "unsafe" ) @@ -995,12 +996,16 @@ const ArrayMaxSize = 4096 // RunMaxSize represents the maximum size of run length encoded containers. const RunMaxSize = 2048 -// container represents a container for uint32 integers. +// container represents a container for uint16 integers. // -// These are used for storing the low bits. Containers are separated into three -// types depending on cardinality. For containers with less than 4,096 values, -// an array or RLE container is used, depending on the contents. For containers -// with more than 4,096 values, the values are encoded into bitmaps. +// These are used for storing the low bits of numbers in larger sets of uint64. +// The high bits are stored in a container's key which is tracked by a separate +// data structure. Integers in a container can be encoded in one of three ways - +// the encoding used is usually whichever is most compact, though any container +// type should be able to encode any set of integers safely. For containers with +// less than 4,096 values, an array is often used. Containers with long runs of +// integers would use run length encoding, and more random data usually uses +// bitmap encoding. type container struct { mapped bool // mapped directly to a byte slice when true containerType byte // array, bitmap, or run @@ -1268,10 +1273,10 @@ func (c *container) contains(v uint16) bool { func (c *container) bitmapCountRuns() (r int) { for i := 0; i < 1023; i++ { v, v1 := c.bitmap[i], c.bitmap[i+1] - r = r + int(popcnt((v<<1)&^v)+((v>>63)&^v1)) + r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1)) } vl := c.bitmap[len(c.bitmap)-1] - r = r + int(popcnt((vl<<1)&^vl)+vl>>63) + r = r + int(popcount((vl<<1)&^vl)+vl>>63) return r } @@ -1663,19 +1668,6 @@ func (c *container) clone() *container { return other } -// flipBitmap returns a new bitmap containter containing the inverse of all -// bits in c. -func (c *container) flipBitmap() *container { - other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - for i, bitmap := range c.bitmap { - other.bitmap[i] = ^bitmap - } - - other.n = other.count() - return other -} - // WriteTo writes c to w. func (c *container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { @@ -1801,6 +1793,43 @@ type ContainerInfo struct { Pointer unsafe.Pointer // offset within the mmap } +// flip returns a new container containing the inverse of all +// bits in a. +func flip(a *container) *container { + if a.isArray() { + return flipArray(a) + } else if a.isRun() { + return flipRun(a) + } else { + return flipBitmap(a) + } +} + +func flipArray(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.arrayToBitmap() + return flipBitmap(x) +} + +func flipBitmap(b *container) *container { + other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + + for i, bitmap := range b.bitmap { + other.bitmap[i] = ^bitmap + } + + other.n = other.count() + return other +} + +func flipRun(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.runToBitmap() + return flipBitmap(x) +} + func intersectionCount(a, b *container) int { if a.isArray() { if b.isArray() { @@ -1900,19 +1929,20 @@ func intersectionCountBitmapRun(a, b *container) (n int) { } func intersectionCountArrayBitmap(a, b *container) (n int) { + ln := len(b.bitmap) for _, val := range a.array { - i := val >> 6 - if i >= uint16(len(b.bitmap)) { + i := int(val >> 6) + if i >= ln { break } off := val % 64 - n += int((b.bitmap[i] & (1 << off)) >> off) + n += int(b.bitmap[i]>>off) & 1 } return n } func intersectionCountBitmapBitmap(a, b *container) (n int) { - return int(popcntAndSlice(a.bitmap, b.bitmap)) + return int(popcountAndSlice(a.bitmap, b.bitmap)) } func intersect(a, b *container) *container { @@ -2055,22 +2085,22 @@ func intersectBitmapRun(a, b *container) *container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b output.bitmap[i] = a.bitmap[i] - output.n += int(popcnt(a.bitmap[i])) + output.n += int(popcount(a.bitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := a.bitmap[i] & mask output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (a.bitmap[i] >> offset) << offset output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (a.bitmap[i] << offset) >> offset output.bitmap[i] |= bits - output.n += int(popcnt(bits)) + output.n += int(popcount(bits)) } // update loop vars i++ @@ -2175,7 +2205,7 @@ func unionArrayArray(a, b *container) *container { // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } output := &container{containerType: ContainerRun} @@ -2232,10 +2262,10 @@ func (c *container) runAppendInterval(v interval16) int { } func unionRunRun(a, b *container) *container { - if a.n == maxContainerVal { + if a.n == maxContainerVal+1 { return a.clone() } - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } na, nb := len(a.runs), len(b.runs) @@ -2266,9 +2296,12 @@ func unionRunRun(a, b *container) *container { } func unionBitmapRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } + if a.n == maxContainerVal+1 { + return a.clone() + } output := a.clone() for j := 0; j < len(b.runs); j++ { output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) @@ -2283,20 +2316,20 @@ func (c *container) bitmapSetRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) - xcnt := popcnt(X) - ycnt := popcnt(Y) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + xcnt := popcount(X) + ycnt := popcount(Y) if x == y { - c.n += int((j - i) - popcnt(c.bitmap[x]&(X&Y))) + c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y))) c.bitmap[x] |= (X & Y) } else { - c.n += int(xcnt - popcnt(c.bitmap[x]&X)) + c.n += int(xcnt - popcount(c.bitmap[x]&X)) c.bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int(64 - popcnt(c.bitmap[i])) + c.n += int(64 - popcount(c.bitmap[i])) c.bitmap[i] = maxBitmap } - c.n += int(ycnt - popcnt(c.bitmap[y]&Y)) + c.n += int(ycnt - popcount(c.bitmap[y]&Y)) c.bitmap[y] |= Y } } @@ -2306,23 +2339,23 @@ func (c *container) bitmapXorRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - cnt := popcnt(c.bitmap[x]) + cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip - c.n += int(popcnt(c.bitmap[x]) - cnt) + c.n += int(popcount(c.bitmap[x]) - cnt) } else { - cnt := popcnt(c.bitmap[x]) + cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= X - c.n += int(popcnt(c.bitmap[x]) - cnt) + c.n += int(popcount(c.bitmap[x]) - cnt) for i := x + 1; i < y; i++ { - cnt = popcnt(c.bitmap[i]) + cnt = popcount(c.bitmap[i]) c.bitmap[i] ^= maxBitmap - c.n += int(popcnt(c.bitmap[i]) - cnt) + c.n += int(popcount(c.bitmap[i]) - cnt) } - cnt = popcnt(c.bitmap[y]) + cnt = popcount(c.bitmap[y]) c.bitmap[y] ^= Y - c.n += int(popcnt(c.bitmap[y]) - cnt) + c.n += int(popcount(c.bitmap[y]) - cnt) } } @@ -2331,18 +2364,18 @@ func (c *container) bitmapZeroRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - c.n -= int(popcnt(c.bitmap[x] & (X & Y))) + c.n -= int(popcount(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) } else { - c.n -= int(popcnt(c.bitmap[x] & X)) + c.n -= int(popcount(c.bitmap[x] & X)) c.bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int(popcnt(c.bitmap[i])) + c.n -= int(popcount(c.bitmap[i])) c.bitmap[i] = 0 } - c.n -= int(popcnt(c.bitmap[y] & Y)) + c.n -= int(popcount(c.bitmap[y] & Y)) c.bitmap[y] &= ^Y } } @@ -2367,7 +2400,7 @@ func unionBitmapBitmap(a, b *container) *container { for i := 0; i < bitmapN; i++ { v := a.bitmap[i] | b.bitmap[i] output.bitmap[i] = v - output.n += int(popcnt(v)) + output.n += int(popcount(v)) } return output @@ -2467,10 +2500,18 @@ func differenceArrayRun(a, b *container) *container { if i < len(a.array) { // keep all array elements after end of runs - output.array = append(output.array, a.array[i:]...) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += len(a.array[i:]) + // It's possible that output was converted from array to bitmap in output.add() + // so check container type before proceeding. + if output.containerType == ContainerArray { + output.array = append(output.array, a.array[i:]...) + // TODO: consider handling container.n mutations in one place + // like we do with container.add(). + output.n += len(a.array[i:]) + } else { + for _, v := range a.array[i:] { + output.add(v) + } + } } return output } @@ -2498,6 +2539,8 @@ func differenceRunArray(a, b *container) *container { bidx := 0 vb := b.array[bidx] + +RUNLOOP: for _, run := range a.runs { start := run.start for vb < run.start { @@ -2509,6 +2552,9 @@ func differenceRunArray(a, b *container) *container { } for vb >= run.start && vb <= run.last { if vb == start { + if vb == 65535 { // overflow + break RUNLOOP + } start++ bidx++ if bidx >= len(b.array) { @@ -2519,6 +2565,9 @@ func differenceRunArray(a, b *container) *container { } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) output.n += int(vb - start) + if vb == 65535 { // overflow + break RUNLOOP + } start = vb + 1 bidx++ if bidx >= len(b.array) { @@ -2540,7 +2589,7 @@ func differenceRunArray(a, b *container) *container { func differenceRunBitmap(a, b *container) *container { // If a is full, difference is the flip of b. if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { - return b.flipBitmap() + return flipBitmap(b) } output := &container{containerType: ContainerRun} output.n = a.n @@ -2615,7 +2664,7 @@ func differenceRunRun(a, b *container) *container { for apos < alen && bpos < blen { switch { case alast < bstart: - // current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run + // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run output.runs = append(output.runs, interval16{start: astart, last: alast}) apos++ if apos < alen { @@ -2623,7 +2672,7 @@ func differenceRunRun(a, b *container) *container { alast = a.runs[apos].last } case blast < astart: - // current B-run entirely preceeds current A-run: advance to next B-run + // current B-run entirely precedes current A-run: advance to next B-run bpos++ if bpos < blen { bstart = b.runs[bpos].start @@ -2770,7 +2819,9 @@ func xorArrayBitmap(a, b *container) *container { } } - if output.count() < ArrayMaxSize { + // It's possible that output was converted from bitmap to array in output.remove() + // so we only do this conversion if output is still a bitmap container. + if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -2785,7 +2836,7 @@ func xorBitmapBitmap(a, b *container) *container { for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] output.bitmap[i] = v - output.n += int(popcnt(v)) + output.n += int(popcount(v)) } if output.count() < ArrayMaxSize { @@ -2948,35 +2999,7 @@ func search64(a []uint64, value uint64) int { // trailingZeroN returns the number of trailing zeros in v. // v must be greater than zero. func trailingZeroN(v uint64) int { - n := int64(63) - if y := v << 32; y != 0 { - n, v = n-32, y - } - if y := v << 16; y != 0 { - n, v = n-16, y - } - if y := v << 8; y != 0 { - n, v = n-8, y - } - if y := v << 4; y != 0 { - n, v = n-4, y - } - if y := v << 2; y != 0 { - n, v = n-2, y - } - return int(n - int64(v<<1>>63)) -} - -// bit population count, taken from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcount(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 + return bits.TrailingZeros64(v) } // ErrorList represents a list of errors. @@ -3185,7 +3208,7 @@ func xorRunRun(a, b *container) *container { if nb == 0 { return a.clone() } - output := &container{} + output := &container{containerType: ContainerRun} lastI, lastJ := -1, -1 @@ -3266,3 +3289,48 @@ func BitmapsEqual(b, c *Bitmap) error { return nil } + +func popcount(x uint64) uint64 { + return uint64(bits.OnesCount64(x)) +} + +func popcountSlice(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcountMaskSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcountAndSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcountOrSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcountXorSlice(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} + diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go new file mode 100644 index 000000000..417ac7db6 --- /dev/null +++ b/roaring/roaring_helpers_test.go @@ -0,0 +1,298 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +/////////////////////////////////////////////////////////////////////////// + +var containerWidth uint64 = 65536 + +////////////////// array +func arrayEmpty() []uint16 { + return make([]uint16, 0) +} + +func arrayFull() []uint16 { + array := make([]uint16, containerWidth) + for i := 0; i < int(containerWidth); i++ { + array[i] = uint16(i) + } + return array +} + +func arrayFirstBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + return array +} + +func arrayLastBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(65535)) + return array +} + +func arrayFirstBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 1; i < int(containerWidth); i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayLastBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 0; i < int(containerWidth)-1; i++ { + array[i] = uint16(i) + } + return array +} + +func arrayInnerBitsSet() []uint16 { + array := make([]uint16, containerWidth-2) + for i := 1; i < int(containerWidth)-1; i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayOuterBitsSet() []uint16 { + return []uint16{0, 65535} +} + +func arrayOddBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2*i + 1) + } + return array +} + +func arrayEvenBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2 * i) + } + return array +} + +////////////////// bitmap +func bitmapEmpty() []uint64 { + return make([]uint64, bitmapN) +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapFirstBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + return bitmap +} + +func bitmapLastBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitUnset() []uint64 { + bitmap := bitmapFull() + bitmap[0] = 0xFFFFFFFFFFFFFFFE + return bitmap +} + +func bitmapLastBitUnset() []uint64 { + bitmap := bitmapFull() + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapInnerBitsSet() []uint64 { + bitmap := bitmapFull() + bitmap[0] = 0xFFFFFFFFFFFFFFFE + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapOuterBitsSet() []uint64 { + bitmap := bitmapEmpty() + bitmap[0] = 0x0000000000000001 + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapOddBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xAAAAAAAAAAAAAAAA + } + return bitmap +} + +func bitmapEvenBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0x5555555555555555 + } + return bitmap +} + +////////////////// run +func runEmpty() []interval16 { + return make([]interval16, 0) +} + +func runFull() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65535}) + return run +} + +func runFirstBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + return run +} + +func runLastBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runFirstBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65535}) + return run +} + +func runLastBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65534}) + return run +} + +func runInnerBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65534}) + return run +} + +func runOuterBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runOddBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)} + } + return run +} + +func runEvenBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)} + } + return run +} + +/////////////////////////////////////////////////////////////////////////// + +// f is a container function taking either one or two containers as input +// func(a *container) *container +// func(a, b *container) *container +type testOp struct { + f interface{} + x string + y string + exp string +} + +func doContainer(containerType byte, data interface{}) *container { + c := &container{ + containerType: containerType, + } + + switch containerType { + case ContainerArray: + c.array = data.([]uint16) + case ContainerBitmap: + c.bitmap = data.([]uint64) + case ContainerRun: + c.runs = data.([]interval16) + } + c.n = c.count() + + return c +} + +func setupContainerTests() map[byte]map[string]*container { + + cts := make(map[byte]map[string]*container) + + // array containers + cts[ContainerArray] = map[string]*container{ + "empty": doContainer(ContainerArray, arrayEmpty()), + "full": doContainer(ContainerArray, arrayFull()), + "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), + } + + // bitmap containers + cts[ContainerBitmap] = map[string]*container{ + "empty": doContainer(ContainerBitmap, bitmapEmpty()), + "full": doContainer(ContainerBitmap, bitmapFull()), + "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), + } + + // run containers + cts[ContainerRun] = map[string]*container{ + "empty": doContainer(ContainerRun, runEmpty()), + "full": doContainer(ContainerRun, runFull()), + "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), + "lastBitSet": doContainer(ContainerRun, runLastBitSet()), + "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), + "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), + "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), + } + + return cts +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 27636838e..c41eb1dc6 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -18,6 +18,8 @@ import ( "bytes" "fmt" "reflect" + "runtime" + "strings" "testing" ) @@ -1479,6 +1481,16 @@ func TestDifferenceRunArray(t *testing.T) { array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27}, exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}}, }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}}, + array: []uint16{65533, 65534, 65535}, + exp: []interval16{{start: 0, last: 20}}, + }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}}, + array: []uint16{37, 65535}, + exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}}, + }, } for i, test := range tests { a.runs = test.runs @@ -1583,14 +1595,69 @@ func TestDifferenceBitmapRun(t *testing.T) { runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, exp: []uint64{0xFFFF0000FFFFFF0F}, }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}}, + exp: []uint64{0xFFFFFFFFFFFFFF80}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 63}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8}, + }, + { + bitmap: []uint64{0x0000000000000001}, + runs: []interval16{{start: 0, last: 0}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0x8000000000000000}, + runs: []interval16{{start: 63, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xC000000000000000, 0x0000000000000003}, + runs: []interval16{{start: 63, last: 64}}, + exp: []uint64{0x4000000000000000, 0x0000000000000002}, + }, + { + bitmap: []uint64{0x0000000000000000}, + runs: []interval16{{start: 5, last: 7}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: bitmapLastBitSet(), + runs: []interval16{{start: 65535, last: 65535}}, + exp: bitmapEmpty(), + }, + { + bitmap: bitmapFull(), + runs: []interval16{{start: 0, last: 65535}}, + exp: bitmapEmpty(), + }, } for i, test := range tests { for i, v := range test.bitmap { a.bitmap[i] = v } - a.n = a.bitmapCountRange(0, 100) + a.n = a.bitmapCountRange(0, 65536) b.runs = test.runs - b.n = b.runCountRange(0, 100) + b.n = b.runCountRange(0, 65536) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) @@ -1622,12 +1689,12 @@ func TestDifferenceBitmapArray(t *testing.T) { exp: []uint16{8, 9, 11, 12, 13, 14, 15}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10}, exp: []uint16{9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{63}, exp: []uint16{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61}, }, @@ -1957,38 +2024,6 @@ func TestXorRunRun(t *testing.T) { } } -func TestBitmapFlip(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - ttable := []struct { - original uint64 - flipped uint64 - }{ - {0x0000000000000000, 0xFFFFFFFFFFFFFFFF}, - {0xFFFFFFFFFFFFFFFF, 0x0000000000000000}, - {0xFFFFFFFFFFFFFFF0, 0x000000000000000F}, - {0xFFFFFFEFFFFFFFFF, 0x0000001000000000}, - {0x0000001000000000, 0xFFFFFFEFFFFFFFFF}, - } - - expectedN := int(65536) - for i, tt := range ttable { - c.bitmap[i] = tt.original - expectedN -= int(popcount(tt.original)) - } - - o := c.flipBitmap() - - for i, tt := range ttable { - if o.bitmap[i] != tt.flipped { - t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i]) - } - } - if o.n != expectedN { - t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n) - } -} - func TestBitmapXorRange(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { @@ -2432,7 +2467,7 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { } } -func TestSearc64(t *testing.T) { +func TestSearch64(t *testing.T) { tests := []struct { a []uint64 value uint64 @@ -2547,12 +2582,12 @@ func TestIntersectArrayBitmap(t *testing.T) { }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), exp: []uint16{1, 63, 543, 639, 65535}, }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapEvens(), + bitmap: bitmapEvenBitsSet(), exp: []uint16{0, 120, 12000, 65534}, }, } @@ -2574,24 +2609,6 @@ func TestIntersectArrayBitmap(t *testing.T) { } } -func bitmapOdds() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xAAAAAAAAAAAAAAAA - } - return bitmap -} - -func bitmapEvens() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0x5555555555555555 - } - return bitmap -} - -var containerWidth uint64 = 65536 - // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the @@ -2673,3 +2690,548 @@ func bitmapVariousContainers() *Bitmap { bm.Optimize() return bm } + +/////////////////////////////////////////////////////////////////////////// + +func getFunctionName(i interface{}) string { + x := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + y := strings.Split(x, ".") + y = y[len(y)-1:] + return y[0] +} + +func TestContainerCombinations(t *testing.T) { + + cts := setupContainerTests() + + containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + + // map used for a more descriptive print + cm := map[byte]string{ + ContainerArray: "array", + ContainerBitmap: "bitmap", + ContainerRun: "run", + } + + testOps := []testOp{ + // intersect + {intersect, "empty", "empty", "empty"}, + {intersect, "empty", "full", "empty"}, + {intersect, "empty", "firstBitSet", "empty"}, + {intersect, "empty", "lastBitSet", "empty"}, + {intersect, "empty", "firstBitUnset", "empty"}, + {intersect, "empty", "lastBitUnset", "empty"}, + {intersect, "empty", "innerBitsSet", "empty"}, + {intersect, "empty", "outerBitsSet", "empty"}, + {intersect, "empty", "oddBitsSet", "empty"}, + {intersect, "empty", "evenBitsSet", "empty"}, + // + {intersect, "full", "empty", "empty"}, + {intersect, "full", "full", "full"}, + {intersect, "full", "firstBitSet", "firstBitSet"}, + {intersect, "full", "lastBitSet", "lastBitSet"}, + {intersect, "full", "firstBitUnset", "firstBitUnset"}, + {intersect, "full", "lastBitUnset", "lastBitUnset"}, + {intersect, "full", "innerBitsSet", "innerBitsSet"}, + {intersect, "full", "outerBitsSet", "outerBitsSet"}, + {intersect, "full", "oddBitsSet", "oddBitsSet"}, + {intersect, "full", "evenBitsSet", "evenBitsSet"}, + // + {intersect, "firstBitSet", "empty", "empty"}, + {intersect, "firstBitSet", "full", "firstBitSet"}, + {intersect, "firstBitSet", "firstBitSet", "firstBitSet"}, + {intersect, "firstBitSet", "lastBitSet", "empty"}, + {intersect, "firstBitSet", "firstBitUnset", "empty"}, + {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, + {intersect, "firstBitSet", "innerBitsSet", "empty"}, + {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + {intersect, "firstBitSet", "oddBitsSet", "empty"}, + {intersect, "firstBitSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "lastBitSet", "empty", "empty"}, + {intersect, "lastBitSet", "full", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitSet", "empty"}, + {intersect, "lastBitSet", "lastBitSet", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitUnset", "lastBitSet"}, + {intersect, "lastBitSet", "lastBitUnset", "empty"}, + {intersect, "lastBitSet", "innerBitsSet", "empty"}, + {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "oddBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "evenBitsSet", "empty"}, + // + {intersect, "firstBitUnset", "empty", "empty"}, + {intersect, "firstBitUnset", "full", "firstBitUnset"}, + {intersect, "firstBitUnset", "firstBitSet", "empty"}, + {intersect, "firstBitUnset", "lastBitSet", "lastBitSet"}, + {intersect, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, + {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + {intersect, "firstBitUnset", "oddBitsSet", "oddBitsSet"}, + //{intersect, "firstBitUnset", "evenBitsSet", ""}, + // + {intersect, "lastBitUnset", "empty", "empty"}, + {intersect, "lastBitUnset", "full", "lastBitUnset"}, + {intersect, "lastBitUnset", "firstBitSet", "firstBitSet"}, + {intersect, "lastBitUnset", "lastBitSet", "empty"}, + {intersect, "lastBitUnset", "firstBitUnset", "innerBitsSet"}, + {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + //{intersect, "lastBitUnset", "oddBitsSet", ""}, + {intersect, "lastBitUnset", "evenBitsSet", "evenBitsSet"}, + // + {intersect, "innerBitsSet", "empty", "empty"}, + {intersect, "innerBitsSet", "full", "innerBitsSet"}, + {intersect, "innerBitsSet", "firstBitSet", "empty"}, + {intersect, "innerBitsSet", "lastBitSet", "empty"}, + {intersect, "innerBitsSet", "firstBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + //{intersect, "innerBitsSet", "oddBitsSet", ""}, + //{intersect, "innerBitsSet", "evenBitsSet", ""}, + // + {intersect, "outerBitsSet", "empty", "empty"}, + {intersect, "outerBitsSet", "full", "outerBitsSet"}, + {intersect, "outerBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "outerBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "outerBitsSet", "firstBitUnset", "lastBitSet"}, + {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, + {intersect, "outerBitsSet", "innerBitsSet", "empty"}, + {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + {intersect, "outerBitsSet", "oddBitsSet", "lastBitSet"}, + {intersect, "outerBitsSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "oddBitsSet", "empty", "empty"}, + {intersect, "oddBitsSet", "full", "oddBitsSet"}, + {intersect, "oddBitsSet", "firstBitSet", "empty"}, + {intersect, "oddBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "oddBitsSet", "firstBitUnset", "oddBitsSet"}, + //{intersect, "oddBitsSet", "lastBitUnset", ""}, + //{intersect, "oddBitsSet", "innerBitsSet", ""}, + {intersect, "oddBitsSet", "outerBitsSet", "lastBitSet"}, + {intersect, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {intersect, "oddBitsSet", "evenBitsSet", "empty"}, + // + {intersect, "evenBitsSet", "empty", "empty"}, + {intersect, "evenBitsSet", "full", "evenBitsSet"}, + {intersect, "evenBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "evenBitsSet", "lastBitSet", "empty"}, + //{intersect, "evenBitsSet", "firstBitUnset", ""}, + {intersect, "evenBitsSet", "lastBitUnset", "evenBitsSet"}, + //{intersect, "evenBitsSet", "innerBitsSet", ""}, + {intersect, "evenBitsSet", "outerBitsSet", "firstBitSet"}, + {intersect, "evenBitsSet", "oddBitsSet", "empty"}, + {intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, + + // union + {union, "empty", "empty", "empty"}, + {union, "empty", "full", "full"}, + {union, "empty", "firstBitSet", "firstBitSet"}, + {union, "empty", "lastBitSet", "lastBitSet"}, + {union, "empty", "firstBitUnset", "firstBitUnset"}, + {union, "empty", "lastBitUnset", "lastBitUnset"}, + {union, "empty", "innerBitsSet", "innerBitsSet"}, + {union, "empty", "outerBitsSet", "outerBitsSet"}, + {union, "empty", "oddBitsSet", "oddBitsSet"}, + {union, "empty", "evenBitsSet", "evenBitsSet"}, + // + {union, "full", "empty", "full"}, + {union, "full", "full", "full"}, + {union, "full", "firstBitSet", "full"}, + {union, "full", "lastBitSet", "full"}, + {union, "full", "firstBitUnset", "full"}, + {union, "full", "lastBitUnset", "full"}, + {union, "full", "innerBitsSet", "full"}, + {union, "full", "outerBitsSet", "full"}, + {union, "full", "oddBitsSet", "full"}, + {union, "full", "evenBitsSet", "full"}, + // + {union, "firstBitSet", "empty", "firstBitSet"}, + {union, "firstBitSet", "full", "full"}, + {union, "firstBitSet", "firstBitSet", "firstBitSet"}, + {union, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {union, "firstBitSet", "firstBitUnset", "full"}, + {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, + {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + //{union, "firstBitSet", "oddBitsSet", ""}, + {union, "firstBitSet", "evenBitsSet", "evenBitsSet"}, + // + {union, "lastBitSet", "empty", "lastBitSet"}, + {union, "lastBitSet", "full", "full"}, + {union, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {union, "lastBitSet", "lastBitSet", "lastBitSet"}, + {union, "lastBitSet", "firstBitUnset", "firstBitUnset"}, + {union, "lastBitSet", "lastBitUnset", "full"}, + {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + {union, "lastBitSet", "oddBitsSet", "oddBitsSet"}, + //{union, "lastBitSet", "evenBitsSet", ""}, + // + {union, "firstBitUnset", "empty", "firstBitUnset"}, + {union, "firstBitUnset", "full", "full"}, + {union, "firstBitUnset", "firstBitSet", "full"}, + {union, "firstBitUnset", "lastBitSet", "firstBitUnset"}, + {union, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {union, "firstBitUnset", "lastBitUnset", "full"}, + {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "outerBitsSet", "full"}, + {union, "firstBitUnset", "oddBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "evenBitsSet", "full"}, + // + {union, "lastBitUnset", "empty", "lastBitUnset"}, + {union, "lastBitUnset", "full", "full"}, + {union, "lastBitUnset", "firstBitSet", "lastBitUnset"}, + {union, "lastBitUnset", "lastBitSet", "full"}, + {union, "lastBitUnset", "firstBitUnset", "full"}, + {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, + {union, "lastBitUnset", "outerBitsSet", "full"}, + {union, "lastBitUnset", "oddBitsSet", "full"}, + {union, "lastBitUnset", "evenBitsSet", "lastBitUnset"}, + // + {union, "innerBitsSet", "empty", "innerBitsSet"}, + {union, "innerBitsSet", "full", "full"}, + {union, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {union, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {union, "innerBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {union, "innerBitsSet", "outerBitsSet", "full"}, + {union, "innerBitsSet", "oddBitsSet", "firstBitUnset"}, + {union, "innerBitsSet", "evenBitsSet", "lastBitUnset"}, + // + {union, "outerBitsSet", "empty", "outerBitsSet"}, + {union, "outerBitsSet", "full", "full"}, + {union, "outerBitsSet", "firstBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "lastBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "firstBitUnset", "full"}, + {union, "outerBitsSet", "lastBitUnset", "full"}, + {union, "outerBitsSet", "innerBitsSet", "full"}, + {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + //{union, "outerBitsSet", "oddBitsSet", ""}, + //{union, "outerBitsSet", "evenBitsSet", ""}, + // + {union, "oddBitsSet", "empty", "oddBitsSet"}, + {union, "oddBitsSet", "full", "full"}, + //{union, "oddBitsSet", "firstBitSet", ""}, + {union, "oddBitsSet", "lastBitSet", "oddBitsSet"}, + {union, "oddBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "oddBitsSet", "lastBitUnset", "full"}, + {union, "oddBitsSet", "innerBitsSet", "firstBitUnset"}, + //{union, "oddBitsSet", "outerBitsSet", ""}, + {union, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {union, "oddBitsSet", "evenBitsSet", "full"}, + // + {union, "evenBitsSet", "empty", "evenBitsSet"}, + {union, "evenBitsSet", "full", "full"}, + {union, "evenBitsSet", "firstBitSet", "evenBitsSet"}, + //{union, "evenBitsSet", "lastBitSet", ""}, + {union, "evenBitsSet", "firstBitUnset", "full"}, + {union, "evenBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "evenBitsSet", "innerBitsSet", "lastBitUnset"}, + //{union, "evenBitsSet", "outerBitsSet", ""}, + {union, "evenBitsSet", "oddBitsSet", "full"}, + {union, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, + + // difference + {difference, "empty", "empty", "empty"}, + {difference, "empty", "full", "empty"}, + {difference, "empty", "firstBitSet", "empty"}, + {difference, "empty", "lastBitSet", "empty"}, + {difference, "empty", "firstBitUnset", "empty"}, + {difference, "empty", "lastBitUnset", "empty"}, + {difference, "empty", "innerBitsSet", "empty"}, + {difference, "empty", "outerBitsSet", "empty"}, + {difference, "empty", "oddBitsSet", "empty"}, + {difference, "empty", "evenBitsSet", "empty"}, + // + {difference, "full", "empty", "full"}, + {difference, "full", "full", "empty"}, + {difference, "full", "firstBitSet", "firstBitUnset"}, + {difference, "full", "lastBitSet", "lastBitUnset"}, + {difference, "full", "firstBitUnset", "firstBitSet"}, + {difference, "full", "lastBitUnset", "lastBitSet"}, + {difference, "full", "innerBitsSet", "outerBitsSet"}, + {difference, "full", "outerBitsSet", "innerBitsSet"}, + {difference, "full", "oddBitsSet", "evenBitsSet"}, + {difference, "full", "evenBitsSet", "oddBitsSet"}, + // + {difference, "firstBitSet", "empty", "firstBitSet"}, + {difference, "firstBitSet", "full", "empty"}, + {difference, "firstBitSet", "firstBitSet", "empty"}, + {difference, "firstBitSet", "lastBitSet", "firstBitSet"}, + {difference, "firstBitSet", "firstBitUnset", "firstBitSet"}, + {difference, "firstBitSet", "lastBitUnset", "empty"}, + {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "outerBitsSet", "empty"}, + {difference, "firstBitSet", "oddBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "evenBitsSet", "empty"}, + // + {difference, "lastBitSet", "empty", "lastBitSet"}, + {difference, "lastBitSet", "full", "empty"}, + {difference, "lastBitSet", "firstBitSet", "lastBitSet"}, + {difference, "lastBitSet", "lastBitSet", "empty"}, + {difference, "lastBitSet", "firstBitUnset", "empty"}, + {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, + {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, + {difference, "lastBitSet", "outerBitsSet", "empty"}, + {difference, "lastBitSet", "oddBitsSet", "empty"}, + {difference, "lastBitSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "firstBitUnset", "empty", "firstBitUnset"}, + {difference, "firstBitUnset", "full", "empty"}, + {difference, "firstBitUnset", "firstBitSet", "firstBitUnset"}, + {difference, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {difference, "firstBitUnset", "firstBitUnset", "empty"}, + {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, + {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + //{difference, "firstBitUnset", "oddBitsSet", ""}, + {difference, "firstBitUnset", "evenBitsSet", "oddBitsSet"}, + // + {difference, "lastBitUnset", "empty", "lastBitUnset"}, + {difference, "lastBitUnset", "full", "empty"}, + {difference, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {difference, "lastBitUnset", "lastBitSet", "lastBitUnset"}, + {difference, "lastBitUnset", "firstBitUnset", "firstBitSet"}, + {difference, "lastBitUnset", "lastBitUnset", "empty"}, + {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + {difference, "lastBitUnset", "oddBitsSet", "evenBitsSet"}, + //{difference, "lastBitUnset", "evenBitsSet", ""}, + // + {difference, "innerBitsSet", "empty", "innerBitsSet"}, + {difference, "innerBitsSet", "full", "empty"}, + {difference, "innerBitsSet", "firstBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "lastBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "firstBitUnset", "empty"}, + {difference, "innerBitsSet", "lastBitUnset", "empty"}, + {difference, "innerBitsSet", "innerBitsSet", "empty"}, + {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + //{difference, "innerBitsSet", "oddBitsSet", ""}, + //{difference, "innerBitsSet", "evenBitsSet", ""}, + // + {difference, "outerBitsSet", "empty", "outerBitsSet"}, + {difference, "outerBitsSet", "full", "empty"}, + {difference, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {difference, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {difference, "outerBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, + {difference, "outerBitsSet", "outerBitsSet", "empty"}, + {difference, "outerBitsSet", "oddBitsSet", "firstBitSet"}, + {difference, "outerBitsSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "oddBitsSet", "empty", "oddBitsSet"}, + {difference, "oddBitsSet", "full", "empty"}, + {difference, "oddBitsSet", "firstBitSet", "oddBitsSet"}, + //{difference, "oddBitsSet", "lastBitSet", ""}, + {difference, "oddBitsSet", "firstBitUnset", "empty"}, + {difference, "oddBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "oddBitsSet", "innerBitsSet", "lastBitSet"}, + //{difference, "oddBitsSet", "outerBitsSet", ""}, + {difference, "oddBitsSet", "oddBitsSet", "empty"}, + {difference, "oddBitsSet", "evenBitsSet", "oddBitsSet"}, + // + {difference, "evenBitsSet", "empty", "evenBitsSet"}, + {difference, "evenBitsSet", "full", "empty"}, + //{difference, "evenBitsSet", "firstBitSet", ""}, + {difference, "evenBitsSet", "lastBitSet", "evenBitsSet"}, + {difference, "evenBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "evenBitsSet", "lastBitUnset", "empty"}, + {difference, "evenBitsSet", "innerBitsSet", "firstBitSet"}, + //{difference, "evenBitsSet", "outerBitsSet", ""}, + {difference, "evenBitsSet", "oddBitsSet", "evenBitsSet"}, + {difference, "evenBitsSet", "evenBitsSet", "empty"}, + + // xor + {xor, "empty", "empty", "empty"}, + {xor, "empty", "full", "full"}, + {xor, "empty", "firstBitSet", "firstBitSet"}, + {xor, "empty", "lastBitSet", "lastBitSet"}, + {xor, "empty", "firstBitUnset", "firstBitUnset"}, + {xor, "empty", "lastBitUnset", "lastBitUnset"}, + {xor, "empty", "innerBitsSet", "innerBitsSet"}, + {xor, "empty", "outerBitsSet", "outerBitsSet"}, + {xor, "empty", "oddBitsSet", "oddBitsSet"}, + {xor, "empty", "evenBitsSet", "evenBitsSet"}, + // + {xor, "full", "empty", "full"}, + {xor, "full", "full", "empty"}, + {xor, "full", "firstBitSet", "firstBitUnset"}, + {xor, "full", "lastBitSet", "lastBitUnset"}, + {xor, "full", "firstBitUnset", "firstBitSet"}, + {xor, "full", "lastBitUnset", "lastBitSet"}, + {xor, "full", "innerBitsSet", "outerBitsSet"}, + {xor, "full", "outerBitsSet", "innerBitsSet"}, + {xor, "full", "oddBitsSet", "evenBitsSet"}, + {xor, "full", "evenBitsSet", "oddBitsSet"}, + // + {xor, "firstBitSet", "empty", "firstBitSet"}, + {xor, "firstBitSet", "full", "firstBitUnset"}, + {xor, "firstBitSet", "firstBitSet", "empty"}, + {xor, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {xor, "firstBitSet", "firstBitUnset", "full"}, + {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, + {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + //{xor, "firstBitSet", "oddBitsSet", ""}, + //{xor, "firstBitSet", "evenBitsSet", ""}, + // + {xor, "lastBitSet", "empty", "lastBitSet"}, + {xor, "lastBitSet", "full", "lastBitUnset"}, + {xor, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {xor, "lastBitSet", "lastBitSet", "empty"}, + {xor, "lastBitSet", "firstBitUnset", "innerBitsSet"}, + {xor, "lastBitSet", "lastBitUnset", "full"}, + {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + //{xor, "lastBitSet", "oddBitsSet", ""}, + //{xor, "lastBitSet", "evenBitsSet", ""}, + // + {xor, "firstBitUnset", "empty", "firstBitUnset"}, + {xor, "firstBitUnset", "full", "firstBitSet"}, + {xor, "firstBitUnset", "firstBitSet", "full"}, + {xor, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {xor, "firstBitUnset", "firstBitUnset", "empty"}, + {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, + {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + //{xor, "firstBitUnset", "oddBitsSet", ""}, + //{xor, "firstBitUnset", "evenBitsSet", ""}, + // + {xor, "lastBitUnset", "empty", "lastBitUnset"}, + {xor, "lastBitUnset", "full", "lastBitSet"}, + {xor, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {xor, "lastBitUnset", "lastBitSet", "full"}, + {xor, "lastBitUnset", "firstBitUnset", "outerBitsSet"}, + {xor, "lastBitUnset", "lastBitUnset", "empty"}, + {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + //{xor, "lastBitUnset", "oddBitsSet", ""}, + //{xor, "lastBitUnset", "evenBitsSet", ""}, + // + {xor, "innerBitsSet", "empty", "innerBitsSet"}, + {xor, "innerBitsSet", "full", "outerBitsSet"}, + {xor, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {xor, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {xor, "innerBitsSet", "firstBitUnset", "lastBitSet"}, + {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, + {xor, "innerBitsSet", "innerBitsSet", "empty"}, + {xor, "innerBitsSet", "outerBitsSet", "full"}, + //{xor, "innerBitsSet", "oddBitsSet", ""}, + //{xor, "innerBitsSet", "evenBitsSet", ""}, + // + {xor, "outerBitsSet", "empty", "outerBitsSet"}, + {xor, "outerBitsSet", "full", "innerBitsSet"}, + {xor, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {xor, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {xor, "outerBitsSet", "firstBitUnset", "lastBitUnset"}, + {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, + {xor, "outerBitsSet", "innerBitsSet", "full"}, + {xor, "outerBitsSet", "outerBitsSet", "empty"}, + //{xor, "outerBitsSet", "oddBitsSet", ""}, + //{xor, "outerBitsSet", "evenBitsSet", ""}, + // + {xor, "oddBitsSet", "empty", "oddBitsSet"}, + {xor, "oddBitsSet", "full", "evenBitsSet"}, + //{xor, "oddBitsSet", "firstBitSet", ""}, + //{xor, "oddBitsSet", "lastBitSet", ""}, + //{xor, "oddBitsSet", "firstBitUnset", ""}, + //{xor, "oddBitsSet", "lastBitUnset", ""}, + //{xor, "oddBitsSet", "innerBitsSet", ""}, + //{xor, "oddBitsSet", "outerBitsSet", ""}, + {xor, "oddBitsSet", "oddBitsSet", "empty"}, + {xor, "oddBitsSet", "evenBitsSet", "full"}, + // + {xor, "evenBitsSet", "empty", "evenBitsSet"}, + {xor, "evenBitsSet", "full", "oddBitsSet"}, + //{xor, "evenBitsSet", "firstBitSet", ""}, + //{xor, "evenBitsSet", "lastBitSet", ""}, + //{xor, "evenBitsSet", "firstBitUnset", ""}, + //{xor, "evenBitsSet", "lastBitUnset", ""}, + //{xor, "evenBitsSet", "innerBitsSet", ""}, + //{xor, "evenBitsSet", "outerBitsSet", ""}, + {xor, "evenBitsSet", "oddBitsSet", "full"}, + {xor, "evenBitsSet", "evenBitsSet", "empty"}, + + // flip + {flip, "empty", "", "full"}, + {flip, "full", "", "empty"}, + {flip, "firstBitSet", "", "firstBitUnset"}, + {flip, "lastBitSet", "", "lastBitUnset"}, + {flip, "firstBitUnset", "", "firstBitSet"}, + {flip, "lastBitUnset", "", "lastBitSet"}, + {flip, "innerBitsSet", "", "outerBitsSet"}, + {flip, "outerBitsSet", "", "innerBitsSet"}, + {flip, "oddBitsSet", "", "evenBitsSet"}, + {flip, "evenBitsSet", "", "oddBitsSet"}, + } + for _, testOp := range testOps { + for _, x := range containerTypes { + for _, y := range containerTypes { + desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) + ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y]) + exp := testOp.exp + + // Convert to all container types and check result. + for _, ct := range containerTypes { + clone := ret.clone() + if ct == ContainerArray { + if clone.isBitmap() { + clone.bitmapToArray() + } else if clone.isRun() { + clone.runToArray() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + // Because xorRunRun resulting in an empty container returns an array container with a + // nil slice array, then we need to check len() on array first (look for 0). + if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { + t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) + } + } else if ct == ContainerBitmap { + if clone.isArray() { + clone.arrayToBitmap() + } else if clone.isRun() { + clone.runToBitmap() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { + t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) + } + } else if ct == ContainerRun { + if clone.isArray() { + clone.arrayToRun() + } else if clone.isBitmap() { + clone.bitmapToRun() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) { + t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs) + } + } + } + } + } + } +} + +//func getFunc(func(a, b *container) *container, m, n *container) *container { +func runContainerFunc(f interface{}, c ...*container) *container { + switch f.(type) { + case func(*container) *container: + return f.(func(*container) *container)(c[0]) + case func(*container, *container) *container: + return f.(func(a, b *container) *container)(c[0], c[1]) + } + return nil +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index ec31f3226..4cbf17e2e 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -291,7 +291,7 @@ func TestBitmap_Max(t *testing.T) { } } -// Ensure CountRange is correct even if rangekey is prior to inital container. +// Ensure CountRange is correct even if rangekey is prior to initial container. func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) { s := uint64(2009 * 1048576) e := uint64(2010 * 1048576) diff --git a/server.go b/server.go index c8e536c50..c8b631580 100644 --- a/server.go +++ b/server.go @@ -15,35 +15,30 @@ package pilosa import ( - "crypto/tls" - "errors" + "context" "fmt" - "io" "log" "net" "net/http" - "net/url" "os" "os/exec" + "path/filepath" "runtime" "strconv" "strings" "sync" "time" - "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" - "golang.org/x/net/context" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // Default server settings. const ( - DefaultAntiEntropyInterval = 10 * time.Minute - DefaultPollingInterval = 60 * time.Second - DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Ensure Server implements interfaces. @@ -53,179 +48,299 @@ var _ StatusHandler = &Server{} // Server represents a holder wrapped by a running HTTP server. type Server struct { - ln net.Listener - // Close management. wg sync.WaitGroup closing chan struct{} - // Data storage and HTTP interface. - Holder *Holder - Handler *Handler + // Internal + Holder *Holder + Cluster *Cluster + diagnostics *DiagnosticsCollector + executor *Executor + + // External + handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper - RemoteClient *http.Client + remoteClient *http.Client + systemInfo SystemInfo + gcNotifier GCNotifier + NewAttrStore func(string) AttrStore + logger Logger + ln net.Listener - // Cluster configuration. - // Host is replaced with actual host after opening if port is ":0". - Network string - URI *URI - Cluster *Cluster - diagnostics *diagnostics.Diagnostics - ClusterID string - - // Background monitoring intervals. - AntiEntropyInterval time.Duration - PollingInterval time.Duration - MetricInterval time.Duration - DiagnosticInterval time.Duration - - // TLS configuration - TLS *tls.Config - - // Misc options. - MaxWritesPerRequest int - - LogOutput io.Writer - logger *log.Logger + NodeID string + URI URI + antiEntropyInterval time.Duration + metricInterval time.Duration + diagnosticInterval time.Duration + maxWritesPerRequest int defaultClient InternalClient + dataDir string +} + +// ServerOption is a functional option type for pilosa.Server +type ServerOption func(s *Server) error + +func OptServerLogger(l Logger) ServerOption { + return func(s *Server) error { + s.logger = l + return nil + } +} + +func OptServerReplicaN(n int) ServerOption { + return func(s *Server) error { + s.Cluster.ReplicaN = n + return nil + } +} + +func OptServerDataDir(dir string) ServerOption { + return func(s *Server) error { + s.dataDir = dir + return nil + } +} + +func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { + return func(s *Server) error { + s.NewAttrStore = af + s.Holder.NewAttrStore = af + return nil + } +} + +func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.antiEntropyInterval = interval + return nil + } +} + +func OptServerLongQueryTime(dur time.Duration) ServerOption { + return func(s *Server) error { + s.Cluster.LongQueryTime = dur + return nil + } +} + +func OptServerHandler(h *Handler) ServerOption { + return func(s *Server) error { + s.handler = h + return nil + } +} + +func OptServerMaxWritesPerRequest(n int) ServerOption { + return func(s *Server) error { + s.maxWritesPerRequest = n + return nil + } +} + +func OptServerMetricInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.metricInterval = dur + return nil + } +} + +func OptServerSystemInfo(si SystemInfo) ServerOption { + return func(s *Server) error { + s.systemInfo = si + return nil + } +} + +func OptServerGCNotifier(gcn GCNotifier) ServerOption { + return func(s *Server) error { + s.gcNotifier = gcn + return nil + } +} + +func OptServerRemoteClient(c *http.Client) ServerOption { + return func(s *Server) error { + s.executor = NewExecutor(c) + s.remoteClient = c + s.defaultClient = NewInternalHTTPClientFromURI(nil, c) + s.Cluster.RemoteClient = c + return nil + } +} + +func OptServerStatsClient(sc StatsClient) ServerOption { + return func(s *Server) error { + s.Holder.Stats = sc + return nil + } +} + +func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.diagnosticInterval = dur + return nil + } +} + +func OptServerListener(ln net.Listener) ServerOption { + return func(s *Server) error { + s.ln = ln + + return nil + } +} + +func OptServerURI(uri *URI) ServerOption { + return func(s *Server) error { + s.URI = *uri + return nil + } } // NewServer returns a new instance of Server. -func NewServer() *Server { +func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ - closing: make(chan struct{}), - + closing: make(chan struct{}), + Cluster: NewCluster(), Holder: NewHolder(), - Handler: NewHandler(), + handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, - diagnostics: diagnostics.New(DefaultDiagnosticServer), + diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), + systemInfo: NewNopSystemInfo(), - Network: "tcp", + gcNotifier: NopGCNotifier, - AntiEntropyInterval: DefaultAntiEntropyInterval, - PollingInterval: DefaultPollingInterval, - MetricInterval: 0, - DiagnosticInterval: 0, + NewAttrStore: NewNopAttrStore, - LogOutput: os.Stderr, + antiEntropyInterval: time.Minute * 10, + metricInterval: 0, + diagnosticInterval: 0, + + logger: NopLogger, } - s.logger = log.New(s.LogOutput, "", log.LstdFlags) + s.diagnostics.server = s - s.Handler.Holder = s.Holder - return s + for _, opt := range opts { + err := opt(s) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + path, err := expandDirName(s.dataDir) + if err != nil { + return nil, err + } + + s.Holder.Path = path + s.Holder.Logger = s.logger + s.Holder.Stats.SetLogger(s.logger) + + s.Cluster.Path = path + s.Cluster.Logger = s.logger + s.Cluster.Holder = s.Holder + + // update URI port with actual listener port. TODO this should probably be done outside of here. + if s.URI.Port() == 0 { + s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + } + + s.NodeID = s.LoadNodeID() + // Set Cluster Node. + node := &Node{ + ID: s.NodeID, + URI: s.URI, + IsCoordinator: s.Cluster.Coordinator == s.NodeID, + } + s.Cluster.Node = node + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) + + s.executor.Holder = s.Holder + s.executor.Node = node + s.executor.Cluster = s.Cluster + s.executor.MaxWritesPerRequest = s.maxWritesPerRequest + s.handler.API.Executor = s.executor + + return s, nil } // Open opens and initializes the server. func (s *Server) Open() error { - var ln net.Listener - var err error - - // If bind URI has the https scheme, enable TLS - if s.URI.Scheme() == "https" && s.TLS != nil { - ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS) - if err != nil { - return err - } - } else if s.URI.Scheme() == "http" { - // Open HTTP listener to determine port (if specified as :0). - ln, err = net.Listen(s.Network, s.URI.HostPort()) - if err != nil { - return fmt.Errorf("net.Listen: %v", err) - } - } else { - return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme()) + s.logger.Printf("open server") + // s.ln can be configured prior to Open() via s.OpenListener(). + if s.ln == nil { + return errors.New("Must pass a listener option to NewServer") } - s.ln = ln - - if s.URI.Port() == 0 { - // If the port is 0, it is set automatically. - // Find out automatically set port and update the host. - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + // Log startup + err := s.Holder.logStartup() + if err != nil { + log.Println(errors.Wrap(err, "logging startup")) } - // Create local node if no cluster is specified. - if len(s.Cluster.Nodes) == 0 { - s.Cluster.Nodes = []*Node{ - {Scheme: s.URI.Scheme(), Host: s.URI.HostPort()}, - } - } + // Get or create NodeID. - for i, n := range s.Cluster.Nodes { - if s.Cluster.NodeByHost(n.Host) != nil { - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i)) - } - } - - // Open holder. - s.Holder.LogOutput = s.LogOutput - if err := s.Holder.Open(); err != nil { - return fmt.Errorf("opening Holder: %v", err) - } - - if err := s.BroadcastReceiver.Start(s); err != nil { - return fmt.Errorf("starting BroadcastReceiver: %v", err) - } - - // Open NodeSet communication - if err := s.Cluster.NodeSet.Open(); err != nil { - return fmt.Errorf("opening NodeSet: %v", err) - } + // Append the NodeID tag to stats. // Create default HTTP client - s.createDefaultClient(s.RemoteClient) // Create executor for executing queries. - e := NewExecutor(s.RemoteClient) - e.Holder = s.Holder - e.Scheme = s.URI.Scheme() - e.Host = s.URI.HostPort() - e.Cluster = s.Cluster - e.MaxWritesPerRequest = s.MaxWritesPerRequest - s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest + + // Cluster settings. + s.Cluster.Broadcaster = s.Broadcaster + s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.Handler.Broadcaster = s.Broadcaster - s.Handler.BroadcastHandler = s - s.Handler.StatusHandler = s - s.Handler.URI = s.URI - s.Handler.Cluster = s.Cluster - s.Handler.Executor = e - s.Handler.LogOutput = s.LogOutput + s.handler.API.Holder = s.Holder + s.handler.API.Broadcaster = s.Broadcaster + s.handler.API.BroadcastHandler = s + s.handler.API.StatusHandler = s + s.handler.API.Cluster = s.Cluster // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster // Serve HTTP. go func() { - server := &http.Server{Handler: s.Handler} - go func() { - <-s.closing - server.Close() - }() - err := server.Serve(ln) - if err != nil && err.Error() != "http: Server closed" { - s.Logger().Printf("HTTP handler terminated with error: %s\n", err) + err := http.Serve(s.ln, s.handler) + if err != nil { + s.logger.Printf("HTTP handler terminated with error: %s\n", err) } }() - // load local ID - if err := s.Holder.loadLocalID(); err != nil { - s.Logger().Println(err) + // Start the BroadcastReceiver. + if err := s.BroadcastReceiver.Start(s); err != nil { + return fmt.Errorf("starting BroadcastReceiver: %v", err) } - if err := s.loadClusterID(); err != nil { - s.Logger().Println(err) + // Open Cluster management. + if err := s.Cluster.Open(); err != nil { + return fmt.Errorf("opening Cluster: %v", err) } + // Open holder. + if err := s.Holder.Open(); err != nil { + return fmt.Errorf("opening Holder: %v", err) + } + if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + return fmt.Errorf("setting nodeState: %v", err) + } + + // Listen for joining nodes. + // This needs to start after the Holder has opened so that nodes can join + // the cluster without waiting for data to load on the coordinator. Before + // this starts, the joins are queued up in the Cluster.joiningLeavingNodes + // buffered channel. + s.Cluster.ListenForJoins() + // Start background monitoring. - s.wg.Add(4) + s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() go func() { defer s.wg.Done(); s.monitorDiagnostics() }() @@ -241,6 +356,9 @@ func (s *Server) Close() error { if s.ln != nil { s.ln.Close() } + if s.Cluster != nil { + s.Cluster.Close() + } if s.Holder != nil { s.Holder.Close() } @@ -248,6 +366,20 @@ func (s *Server) Close() error { return nil } +// LoadNodeID gets NodeID from disk, or creates a new value. +// If server.NodeID is already set, a new ID is not created. +func (s *Server) LoadNodeID() string { + if s.NodeID != "" { + return s.NodeID + } + nodeID, err := s.Holder.loadNodeID() + if err != nil { + s.logger.Printf("loading NodeID: %v", err) + return s.NodeID + } + return nodeID +} + // Addr returns the address of the listener. func (s *Server) Addr() net.Addr { if s.ln == nil { @@ -255,34 +387,12 @@ func (s *Server) Addr() net.Addr { } return s.ln.Addr() } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} - -// Logger returns a logger that writes to LogOutput -func (s *Server) Logger() *log.Logger { return s.logger } func (s *Server) monitorAntiEntropy() { - ticker := time.NewTicker(s.AntiEntropyInterval) + ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() - s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) for { // Wait for tick or a close. @@ -293,68 +403,30 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.Logger().Printf("holder sync beginning") + s.logger.Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer syncer.Holder = s.Holder - syncer.URI = s.URI + syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.RemoteClient + syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.Logger().Printf("holder sync error: err=%s", err) + s.logger.Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.Logger().Printf("holder sync complete") + s.logger.Printf("holder sync complete") dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } } -// monitorMaxSlices periodically pulls the highest slice from each node in the cluster. -func (s *Server) monitorMaxSlices() { - // Ignore if only one node in the cluster. - if len(s.Cluster.Nodes) <= 1 { - return - } - - ticker := time.NewTicker(s.PollingInterval) - defer ticker.Stop() - - for { - select { - case <-s.closing: - return - case <-ticker.C: - } - - oldmaxslices := s.Holder.MaxSlices() - for _, node := range s.Cluster.Nodes { - if s.URI.HostPort() != node.Host { - maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host) - for index, newmax := range maxSlices { - // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation - if localIndex := s.Holder.Index(index); localIndex != nil { - if newmax > oldmaxslices[index] { - oldmaxslices[index] = newmax - localIndex.SetRemoteMaxSlice(newmax) - } - } else { - s.Logger().Printf("Local Index not found: %s", index) - } - } - } - } - } -} - // ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { @@ -369,10 +441,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { idx.SetRemoteMaxSlice(obj.Slice) } case *internal.CreateIndexMessage: - opt := IndexOptions{ - ColumnLabel: obj.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), - } + opt := IndexOptions{} _, err := s.Holder.CreateIndex(obj.Index, opt) if err != nil { return err @@ -386,16 +455,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := FrameOptions{ - RowLabel: obj.Meta.RowLabel, - InverseEnabled: obj.Meta.InverseEnabled, - RangeEnabled: obj.Meta.RangeEnabled, - CacheType: obj.Meta.CacheType, - CacheSize: obj.Meta.CacheSize, - TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), - Fields: decodeFields(obj.Meta.Fields), - } - _, err := idx.CreateFrame(obj.Frame, opt) + opt := decodeFrameOptions(obj.Meta) + _, err := idx.CreateFrame(obj.Frame, *opt) if err != nil { return err } @@ -404,6 +465,17 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := idx.DeleteFrame(obj.Frame); err != nil { return err } + case *internal.CreateFieldMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + field := decodeField(obj.Field) + if err := f.CreateField(field); err != nil { + return err + } + case *internal.DeleteFieldMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + if err := f.DeleteField(obj.Field); err != nil { + return err + } case *internal.CreateInputDefinitionMessage: idx := s.Holder.Index(obj.Index) if idx == nil { @@ -416,6 +488,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.CreateViewMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + if f == nil { + return fmt.Errorf("Local Frame not found: %s", obj.Frame) + } + _, _, err := f.createViewIfNotExistsBase(obj.View) + if err != nil { + return err + } case *internal.DeleteViewMessage: f := s.Holder.Frame(obj.Index, obj.Frame) if f == nil { @@ -425,7 +506,36 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.ClusterStatus: + err := s.Cluster.MergeClusterStatus(obj) + if err != nil { + return err + } + case *internal.ResizeInstruction: + err := s.Cluster.FollowResizeInstruction(obj) + if err != nil { + return err + } + case *internal.ResizeInstructionComplete: + err := s.Cluster.MarkResizeInstructionComplete(obj) + if err != nil { + return err + } + case *internal.SetCoordinatorMessage: + s.Cluster.SetCoordinator(DecodeNode(obj.New)) + case *internal.UpdateCoordinatorMessage: + s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) + case *internal.NodeStateMessage: + err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) + if err != nil { + return err + } + case *internal.RecalculateCaches: + s.Holder.RecalculateCaches() + case *internal.NodeEventMessage: + s.Cluster.ReceiveEvent(DecodeNodeEvent(obj)) } + return nil } @@ -433,19 +543,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { - uri, err := node.URI() - if err != nil { - return err - } - + node := node + s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. - if *s.URI == *uri { + if s.URI == node.URI { continue } - ctx := context.WithValue(context.Background(), "uri", uri) eg.Go(func() error { - return s.defaultClient.SendMessage(ctx, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) }) } @@ -457,153 +563,147 @@ func (s *Server) SendAsync(pb proto.Message) error { return s.Gossiper.SendAsync(pb) } -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +// SendTo represents an implementation of Broadcaster. +func (s *Server) SendTo(to *Node, pb proto.Message) error { + s.logger.Printf("SendTo: %s", to.URI) + return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) +} + // Server implements StatusHandler. +// LocalStatus is used to periodically sync information +// between nodes. Under normal conditions, nodes should +// remain in sync through Broadcast messages. For cases +// where a node fails to receive a Broadcast message, or +// when a new (empty) node needs to get in sync with the +// rest of the cluster, two things are shared via gossip: +// - MaxSlice/MaxInverseSlice by Index +// - Schema +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalStatus() (proto.Message, error) { + if s.Cluster == nil { + return nil, errors.New("Server.Cluster is nil") + } if s.Holder == nil { return nil, errors.New("Server.Holder is nil") } ns := internal.NodeStatus{ - Scheme: s.URI.Scheme(), - Host: s.URI.HostPort(), - State: NodeStateUp, - Indexes: EncodeIndexes(s.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI.HostPort()) + Node: EncodeNode(s.Cluster.Node), + MaxSlices: s.Holder.EncodeMaxSlices(), + Schema: s.Holder.EncodeSchema(), } return &ns, nil } -// ClusterStatus returns the NodeState for all nodes in the cluster. +// ClusterStatus returns the ClusterState and NodeSet for the cluster. func (s *Server) ClusterStatus() (proto.Message, error) { - // Update local Node.state. - ns, err := s.LocalStatus() - if err != nil { - return nil, err - } - node := s.Cluster.NodeByHost(s.URI.HostPort()) - node.SetStatus(ns.(*internal.NodeStatus)) - - // Update NodeState for all nodes. - for host, nodeState := range s.Cluster.NodeStates() { - // In a default configuration (or single-node) where a StaticNodeSet is used - // then all nodes are marked as DOWN. At the very least, we should consider - // the local node as UP. - // TODO: we should be able to remove this check if/when cluster.Nodes and - // cluster.NodeSet are unified. - if host == s.URI.HostPort() { - nodeState = NodeStateUp - } - node := s.Cluster.NodeByHost(host) - node.SetState(nodeState) - } - return s.Cluster.Status(), nil } -// HandleRemoteStatus receives incoming NodeState from remote nodes. +// HandleRemoteStatus receives incoming NodeStatus from remote nodes. func (s *Server) HandleRemoteStatus(pb proto.Message) error { - return s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + // Ignore NodeStatus messages until the cluster is in a Normal state. + if s.Cluster.State() != ClusterStateNormal { + return nil + } + + go func() { + // Make sure the holder has opened. + <-s.Holder.opened + + err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + if err != nil { + s.logger.Printf("merge remote status: %s", err) + } + }() + + return nil } func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { - // Update Node.state. - node := s.Cluster.NodeByHost(ns.Host) - node.SetStatus(ns) + // Ignore status updates from self. + if s.NodeID == DecodeNode(ns.Node).ID { + return nil + } - // Create indexes that don't exist. - for _, index := range ns.Indexes { - opt := IndexOptions{ - ColumnLabel: index.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(index.Meta.TimeQuantum), + // Sync schema. + if err := s.Holder.ApplySchema(ns.Schema); err != nil { + return err + } + + // Sync maxSlices (standard). + oldmaxslices := s.Holder.MaxSlices() + for index, newMax := range ns.MaxSlices.Standard { + localIndex := s.Holder.Index(index) + // if we don't know about an index locally, log an error because + // indexes should be created and synced prior to slice creation + if localIndex == nil { + s.logger.Printf("Local Index not found: %s", index) + continue } - idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) - if err != nil { - return err + if newMax > oldmaxslices[index] { + oldmaxslices[index] = newMax + localIndex.SetRemoteMaxSlice(newMax) } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - CacheSize: f.Meta.CacheSize, - } - _, err := idx.CreateFrameIfNotExists(f.Name, opt) - if err != nil { - return err - } + } + + // Sync maxSlices (inverse). + oldMaxInverseSlices := s.Holder.MaxInverseSlices() + for index, newMaxInverse := range ns.MaxSlices.Inverse { + localIndex := s.Holder.Index(index) + // if we don't know about an index locally, log an error because + // indexes should be created and synced prior to slice creation + if localIndex == nil { + s.logger.Printf("Local Index not found: %s", index) + continue + } + if newMaxInverse > oldMaxInverseSlices[index] { + oldMaxInverseSlices[index] = newMaxInverse + localIndex.SetRemoteMaxInverseSlice(newMaxInverse) } } return nil } -func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) { - // Create HTTP request. - req, err := http.NewRequest("GET", (&url.URL{ - Scheme: scheme, - Host: hostPort, - Path: "/slices/max", - }).String(), nil) - - if err != nil { - return nil, err - } - - // Require protobuf encoding. - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - nodeURI, err := NewURIFromAddress(hostPort) - if err != nil { - return nil, err - } - nodeURI.SetScheme(scheme) - ctx := context.WithValue(context.Background(), "uri", nodeURI) - return s.defaultClient.MaxSliceByIndex(ctx) -} - // monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { - if s.DiagnosticInterval <= 0 { - s.Logger().Printf("diagnostics disabled") + // Do not send more than once a minute + if s.diagnosticInterval < time.Minute { + s.logger.Printf("diagnostics disabled") return + } else { + s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) } - s.diagnostics.SetLogger(s.LogOutput) + s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) - s.diagnostics.SetInterval(s.DiagnosticInterval) - s.diagnostics.Open() s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ",")) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) - s.diagnostics.Set("LocalID", s.Holder.LocalID) - s.diagnostics.Set("ClusterID", s.ClusterID) + s.diagnostics.Set("NodeID", s.NodeID) + s.diagnostics.Set("ClusterID", s.Cluster.ID) s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder) openFiles, err := CountOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) s.diagnostics.EnrichWithMemoryInfo() + s.diagnostics.EnrichWithSchemaProperties() s.diagnostics.CheckVersion() - s.diagnostics.Flush() + err = s.diagnostics.Flush() + if err != nil { + s.logger.Printf("Diagnostics error: %s", err) + } } - ticker := time.NewTicker(s.DiagnosticInterval) + ticker := time.NewTicker(s.diagnosticInterval) defer ticker.Stop() flush() for { @@ -620,25 +720,24 @@ func (s *Server) monitorDiagnostics() { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { // Disable metrics when poll interval is zero. - if s.MetricInterval <= 0 { + if s.metricInterval <= 0 { return } var m runtime.MemStats - ticker := time.NewTicker(s.MetricInterval) + ticker := time.NewTicker(s.metricInterval) defer ticker.Stop() - gcn := gcnotifier.New() - defer gcn.Close() + defer s.gcNotifier.Close() - s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.logger.Printf("runtime stats initializing (%s interval)", s.metricInterval) for { // Wait for tick or a close. select { case <-s.closing: return - case <-gcn.AfterGC(): + case <-s.gcNotifier.AfterGC(): // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: @@ -663,29 +762,6 @@ func (s *Server) monitorRuntime() { } } -func (s *Server) createDefaultClient(remoteClient *http.Client) { - s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) -} - -func (s *Server) loadClusterID() error { - // If this is the first node in the cluster, set the ClusterID to its ID - node0URI, err := s.Cluster.Nodes[0].URI() - if err == nil { - if s.URI.Equals(node0URI) { - s.ClusterID = s.Holder.LocalID - return nil - } - } else { - return err - } - if clusterID, err := s.defaultClient.NodeID(node0URI); err == nil { - s.ClusterID = clusterID - return nil - } else { - return err - } -} - // CountOpenFiles on operating systems that support lsof. func CountOpenFiles() (int, error) { switch runtime.GOOS { @@ -707,8 +783,8 @@ func CountOpenFiles() (int, error) { } } -// StatusHandler specifies two methods which an object must implement to share -// state in the cluster. These are used by the GossipNodeSet to implement the +// StatusHandler specifies the methods which an object must implement to share +// state in the cluster. These are used by the GossipMemberSet to implement the // LocalState and MergeRemoteState methods of memberlist.Delegate type StatusHandler interface { LocalStatus() (proto.Message, error) @@ -716,38 +792,14 @@ type StatusHandler interface { HandleRemoteStatus(proto.Message) error } -type diagnosticsFrameProperties struct { - BSIFieldCount int - TimeQuantumEnabled bool -} - -func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) { - // NOTE: this function is not in the diagnostics package, since circular imports are not allowed. - var numSlices uint64 - numFrames := 0 - numIndexes := 0 - bsiFieldCount := 0 - timeQuantumEnabled := false - - for _, index := range holder.Indexes() { - numSlices += index.MaxSlice() + 1 - numIndexes += 1 - for _, frame := range index.Frames() { - numFrames += 1 - if frame.rangeEnabled { - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields.Fields) - } - } - if frame.TimeQuantum() != "" { - timeQuantumEnabled = true - } +func expandDirName(path string) (string, error) { + prefix := "~" + string(filepath.Separator) + if strings.HasPrefix(path, prefix) { + HomeDir := os.Getenv("HOME") + if HomeDir == "" { + return "", errors.New("data directory not specified and no home dir available") } + return filepath.Join(HomeDir, strings.TrimPrefix(path, prefix)), nil } - - d.Set("NumIndexes", numIndexes) - d.Set("NumFrames", numFrames) - d.Set("NumSlices", numSlices) - d.Set("BSIFieldCount", bsiFieldCount) - d.Set("TimeQuantumEnabled", timeQuantumEnabled) + return path, nil } diff --git a/server/cluster_test.go b/server/cluster_test.go new file mode 100644 index 000000000..326753a3d --- /dev/null +++ b/server/cluster_test.go @@ -0,0 +1,519 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server_test + +import ( + "context" + "fmt" + "net/http" + "reflect" + "strings" + "testing" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// Ensure program can send/receive broadcast messages. +func TestMain_SendReceiveMessage(t *testing.T) { + ms := test.MustRunMainWithCluster(t, 2) + m0, m1 := ms[0], ms[1] + defer m0.Close() + defer m1.Close() + + m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) + m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + // Expected indexes and Frames + expected := map[string][]string{ + "i": []string{"f"}, + } + + // Create a client for each node. + client0 := m0.Client() + client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Make sure node0 knows about the index and frame created. + schema0, err := client0.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received0 := map[string][]string{} + for _, idx := range schema0 { + received0[idx.Name] = []string{} + for _, frame := range idx.Frames { + received0[idx.Name] = append(received0[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received0, expected) { + t.Fatalf("unexpected schema on node0: %s", received0) + } + + // Make sure node1 knows about the index and frame created. + schema1, err := client1.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received1 := map[string][]string{} + for _, idx := range schema1 { + received1[idx.Name] = []string{} + for _, frame := range idx.Frames { + received1[idx.Name] = append(received1[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received1, expected) { + t.Fatalf("unexpected schema on node1: %s", received1) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) + `); err != nil { + t.Fatal(err) + } + + // We have to wait for the broadcast message to be sent before checking state. + time.Sleep(1 * time.Second) + + // Make sure node0 knows about the latest MaxSlice. + maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices0["i"] != 2 { + t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + } + + // Make sure node1 knows about the latest MaxSlice. + maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices1["i"] != 2 { + t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "col", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + // We have to wait for the broadcast message to be sent before checking state. + time.Sleep(1 * time.Second) + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } +} + +// Ensure that an empty node comes up in a NORMAL state. +func TestClusterResize_EmptyNode(t *testing.T) { + m0 := test.MustRunMain() + defer m0.Close() + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State()) + } +} + +// Ensure that a cluster of empty nodes comes up in a NORMAL state. +func TestClusterResize_EmptyNodes(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + gossipHost := "localhost" + gossipPort := 0 + seed, err := m0.RunWithTransport(gossipHost, gossipPort, []string{}) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + seed, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed}) + if err != nil { + t.Fatal(err) + } + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } +} + +// Ensure that adding a node correctly resizes the cluster. +func TestClusterResize_AddNode(t *testing.T) { + t.Run("NoData", func(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + seed, err := m0.RunWithTransport("localhost", 0, []string{}) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } + }) + t.Run("WithIndex", func(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + seed, err := m0.RunWithTransport("localhost", 0, []string{}) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } + }) + t.Run("ContinuousSlices", func(t *testing.T) { + + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + seed, err := m0.RunWithTransport("localhost", 0, []string{}) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + //client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=1300000) + `); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } + }) + t.Run("SkippedSlice", func(t *testing.T) { + + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + seed, err := m0.RunWithTransport("localhost", 0, []string{}) + if err != nil { + t.Fatal(err) + } + + // Create a client for each node. + client0 := m0.Client() + //client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Write data on first node. Note that no data is placed on slice 1. + if _, err := m0.Query("i", "", ` + SetBit(row=1, frame="f", col=1) + SetBit(row=1, frame="f", col=2400000) + `); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + _, err = m1.RunWithTransport("localhost", 0, []string{seed}) + if err != nil { + return err + } + return nil + }) + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } + }) +} + +// Ensure that redundant gossip seeds are used +func TestCluster_GossipMembership(t *testing.T) { + t.Run("Node0Down", func(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster(true) + defer m0.Close() + + seed, err := m0.RunWithTransport("localhost", 0, []string{}) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster(false) + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + // Pass invalid seed as first in list + _, err := m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) + if err != nil { + return err + } + return nil + }) + + // Configure node2 + m2 := test.NewMainWithCluster(false) + defer m2.Close() + + eg.Go(func() error { + // Pass invalid seed as last in list + _, err := m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) + if err != nil { + return err + } + return nil + }) + + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State()) + } + + numNodes := len(m0.Server.Cluster.Status().Nodes) + if numNodes != 3 { + t.Fatalf("Expected 3 nodes, got %d", numNodes) + } + }) +} + +func TestClusterResize_RemoveNode(t *testing.T) { + cluster := test.MustRunMainWithCluster(t, 3) + m0 := cluster[0] + m1 := cluster[1] + + t.Run("ErrorRemoveInvalidNode", func(t *testing.T) { + resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`) + expBody := "removing node: finding node to remove: node with provided ID does not exist" + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected StatusCode %d but got %d", http.StatusNotFound, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) + + t.Run("ErrorRemoveCoordinator", func(t *testing.T) { + resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + + expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator." + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) + + t.Run("ErrorRemoveOnNonCoordinator", func(t *testing.T) { + resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "") + coordinatorNodeID := resp.Body + + resp = test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + + expBody := fmt.Sprintf("removing node: calling node leave: node removal requests are only valid on the coordinator node: %s", coordinatorNodeID) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if strings.TrimSpace(resp.Body) != expBody { + t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) + + t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) { + client0 := m0.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // This is an attempt to ensure there is data on both nodes, but is not guaranteed. + // TODO: Deterministic node IDs would ensure consistent results + setBits := "" + for i := 0; i < 20; i++ { + setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + } + + if _, err := m0.Query("i", "", setBits); err != nil { + t.Fatal(err) + } + + resp := test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "") + nodeID := resp.Body + + resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) + expBody := "not enough data to perform resize" + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) + } else if !strings.Contains(resp.Body, expBody) { + t.Fatalf("expected to contain '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) + } + }) +} diff --git a/server/config.go b/server/config.go new file mode 100644 index 000000000..2e2feeeb2 --- /dev/null +++ b/server/config.go @@ -0,0 +1,132 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "time" + + "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/toml" +) + +// Cluster types. +const ( + ClusterNone = "" + ClusterStatic = "static" + ClusterGossip = "gossip" +) + +// TLSConfig contains TLS configuration +type TLSConfig struct { + // CertificatePath contains the path to the certificate (.crt or .pem file) + CertificatePath string `toml:"certificate-path"` + // CertificateKeyPath contains the path to the certificate key (.key file) + CertificateKeyPath string `toml:"certificate-key-path"` + // SkipVerify disables verification for self-signed certificates + SkipVerify bool `toml:"skip-verify"` +} + +// Config represents the configuration for the command. +type Config struct { + // DataDir is the directory where Pilosa stores both indexed data and + // running state such as cluster topology information. + DataDir string `toml:"data-dir"` + // Bind is the host:port on which Pilosa will listen. + Bind string `toml:"bind"` + + // MaxWritesPerRequest limits the number of mutating commands that can be in + // a single request to the server. This includes SetBit, ClearBit, + // SetRowAttrs & SetColumnAttrs. + MaxWritesPerRequest int `toml:"max-writes-per-request"` + + // LogPath configures where Pilosa will write logs. + LogPath string `toml:"log-path"` + + // Verbose toggles verbose logging which can be useful for debugging. + Verbose bool `toml:"verbose"` + + // TLS + TLS TLSConfig + + Cluster struct { + // Disabled controls whether clustering functionality is enabled. + Disabled bool `toml:"disabled"` + Coordinator bool `toml:"coordinator"` + ReplicaN int `toml:"replicas"` + Hosts []string `toml:"hosts"` + LongQueryTime toml.Duration `toml:"long-query-time"` + } `toml:"cluster"` + + // Gossip config is based around memberlist.Config. + Gossip gossip.Config `toml:"gossip"` + + AntiEntropy struct { + Interval toml.Duration `toml:"interval"` + } `toml:"anti-entropy"` + + Metric struct { + // Service can be statsd, expvar, or none. + Service string `toml:"service"` + // Host tells the statsd client where to write. + Host string `toml:"host"` + PollInterval toml.Duration `toml:"poll-interval"` + // Diagnostics toggles sending some limited diagnostic information to + // Pilosa's developers. + Diagnostics bool `toml:"diagnostics"` + } `toml:"metric"` +} + +// NewConfig returns an instance of Config with default options. +func NewConfig() *Config { + c := &Config{ + DataDir: "~/.pilosa", + Bind: ":10101", + MaxWritesPerRequest: 5000, + // LogPath: "", + // Verbose: false, + TLS: TLSConfig{}, + } + + // Cluster config. + c.Cluster.Disabled = false + // c.Cluster.Coordinator = false + c.Cluster.ReplicaN = 1 + c.Cluster.Hosts = []string{} + c.Cluster.LongQueryTime = toml.Duration(time.Minute) + + // Gossip config. + c.Gossip.Port = "14000" + // c.Gossip.Seeds = []string{} + // c.Gossip.Key = "" + c.Gossip.StreamTimeout = toml.Duration(10 * time.Second) + c.Gossip.SuspicionMult = 4 + c.Gossip.PushPullInterval = toml.Duration(30 * time.Second) + c.Gossip.ProbeInterval = toml.Duration(1 * time.Second) + c.Gossip.ProbeTimeout = toml.Duration(500 * time.Millisecond) + c.Gossip.Interval = toml.Duration(200 * time.Millisecond) + c.Gossip.Nodes = 3 + c.Gossip.ToTheDeadTime = toml.Duration(30 * time.Second) + + // AntiEntropy config. + c.AntiEntropy.Interval = toml.Duration(10 * time.Minute) + + // Metric config. + c.Metric.Service = "none" + // c.Metric.Host = "" + c.Metric.PollInterval = toml.Duration(0 * time.Minute) + c.Metric.Diagnostics = true + + return c +} diff --git a/config_test.go b/server/config_test.go similarity index 64% rename from config_test.go rename to server/config_test.go index dbeba89a0..2027ef35a 100644 --- a/config_test.go +++ b/server/config_test.go @@ -12,46 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package server_test import ( "reflect" "testing" "time" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" ) func Test_NewConfig(t *testing.T) { - c := pilosa.NewConfig() + c := server.NewConfig() - c.Cluster.Hosts = []string{c.Bind, "localhost:10102"} - - // Change cluster type from the default (gossip) to an invalid string. - c.Cluster.Type = "invalid-type" - if err := c.Validate(); err != pilosa.ErrConfigClusterTypeInvalid { - t.Fatal(err) - } - - // Change cluster type back to gossip. - c.Cluster.Type = pilosa.ClusterGossip - - // Check for bind address in cluster hosts. - c.Bind = "localhost:1" - if err := c.Validate(); err != pilosa.ErrConfigHostsMissing { - t.Fatal(err) - } - - c.Bind = "localhost:10101" - c.Cluster.ReplicaN = 2 - c.GossipSeed = "localhost:14000" - if err := c.Validate(); err != nil { - t.Fatal(err) + if c.Cluster.Disabled { + t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) } } func TestDuration(t *testing.T) { - d := pilosa.Duration(time.Second * 182) + d := toml.Duration(time.Second * 182) if d.String() != "3m2s" { t.Fatalf("Unexpected time Duration %s", d) } diff --git a/server/server.go b/server/server.go index d706d83f1..eb660fc23 100644 --- a/server/server.go +++ b/server/server.go @@ -20,59 +20,66 @@ package server import ( - "errors" "fmt" "io" + "log" "math/rand" + "net" + "net/http" "os" - "path/filepath" + "os/signal" "strconv" - "strings" + "syscall" "time" "crypto/tls" - "io/ioutil" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/gcnotify" + "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/statsd" + "github.com/pkg/errors" ) func init() { rand.Seed(time.Now().UTC().UnixNano()) } -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" -) +type loggerLogger interface { + pilosa.Logger + Logger() *log.Logger +} // Command represents the state of the pilosa server command. type Command struct { Server *pilosa.Server // Configuration. - Config *pilosa.Config + Config *Config - // Profiling options. - CPUProfile string - CPUTime time.Duration + // Gossip transport + GossipTransport *gossip.Transport // Standard input/output *pilosa.CmdIO - // running will be closed once Command.Run is finished. + // Started will be closed once Command.Run is finished. Started chan struct{} // Done will be closed when Command.Close() is called Done chan struct{} + + // Passed to the Gossip implementation. + logOutput io.Writer + logger loggerLogger } // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { return &Command{ - Server: pilosa.NewServer(), - Config: pilosa.NewConfig(), + Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), @@ -81,83 +88,87 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { } } -// Run executes the pilosa server. -func (m *Command) Run(args ...string) (err error) { +// Start starts the pilosa server - it returns once the server is running. +func (m *Command) Start() (err error) { defer close(m.Started) - prefix := "~" + string(filepath.Separator) - if strings.HasPrefix(m.Config.DataDir, prefix) { - HomeDir := os.Getenv("HOME") - if HomeDir == "" { - return errors.New("data directory not specified and no home dir available") - } - m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) - } // SetupServer err = m.SetupServer() if err != nil { - return err + return errors.Wrap(err, "setting up server") + } + + // SetupNetworking + err = m.SetupNetworking() + if err != nil { + return errors.Wrap(err, "setting up networking") } // Initialize server. if err = m.Server.Open(); err != nil { - return fmt.Errorf("server.Open: %v", err) + return errors.Wrap(err, "opening server") } - m.Server.Logger().Printf("Listening as %s\n", m.Server.URI.Normalize()) + m.logger.Printf("Listening as %s\n", m.Server.URI) + + return nil +} + +// Wait waits for the server to be closed or interrupted. +func (m *Command) Wait() error { + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + select { + case sig := <-c: + m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + return errors.Wrap(m.Close(), "closing command") + case <-m.Done: + m.logger.Printf("Server closed externally") + return nil + } +} + +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { + var err error + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrap(err, "opening file") + } + } + + if m.Config.Verbose { + m.logger = pilosa.NewVerboseLogger(m.logOutput) + } else { + m.logger = pilosa.NewStandardLogger(m.logOutput) + } return nil } // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - err := m.Config.Validate() + err := m.setupLogger() if err != nil { - return err + return errors.Wrap(err, "setting up logger") } + handler := pilosa.NewHandler() + handler.Logger = m.logger + handler.FileSystem = &statik.FileSystem{} + handler.API = pilosa.NewAPI() + handler.API.Logger = m.logger + uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { - return err + return errors.Wrap(err, "processing bind address") } - m.Server.URI = uri - - cluster := pilosa.NewCluster() - cluster.ReplicaN = m.Config.Cluster.ReplicaN - - for _, address := range m.Config.Cluster.Hosts { - uri, err := pilosa.NewURIFromAddress(address) - if err != nil { - return err - } - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - }) - } - m.Server.Cluster = cluster - - // Setup logging output. - m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr) - if err != nil { - return err - } - - // Configure holder. - m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir) - m.Server.Holder.Path = m.Config.DataDir - m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) - if m.Config.Metric.Diagnostics { - m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) - } - m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) - if err != nil { - return err - } - - m.Server.Holder.Stats.SetLogger(m.Server.LogOutput) - - // Copy configuration flags. - m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest // Setup TLS var TLSConfig *tls.Config @@ -170,102 +181,150 @@ func (m *Command) SetupServer() error { } cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath) if err != nil { - return err + return errors.Wrap(err, "load x509 key pair") } - m.Server.TLS = &tls.Config{ + TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: m.Config.TLS.SkipVerify, } - - // TODO Review this location - - TLSConfig = m.Server.TLS - - } - c := pilosa.GetHTTPClient(TLSConfig) - m.Server.RemoteClient = c - m.Server.Handler.RemoteClient = c - - // Set internal port (string). - gossipPortStr := pilosa.DefaultGossipPort - // Config.GossipPort is deprecated, so Config.Gossip.Port has priority - if m.Config.Gossip.Port != "" { - gossipPortStr = m.Config.Gossip.Port - } else if m.Config.GossipPort != "" { - gossipPortStr = m.Config.GossipPort } - switch m.Config.Cluster.Type { - case pilosa.ClusterGossip: - gossipPort, err := strconv.Atoi(gossipPortStr) - if err != nil { - return err - } - gossipSeed := pilosa.DefaultHost + ":" + pilosa.DefaultGossipPort - // Config.GossipSeed is deprecated, so Config.Gossip.Seed has priority - if m.Config.Gossip.Seed != "" { - gossipSeed = m.Config.Gossip.Seed - } else if m.Config.GossipSeed != "" { - gossipSeed = m.Config.GossipSeed - } - - var gossipKey []byte - if m.Config.Gossip.Key != "" { - gossipKey, err = ioutil.ReadFile(m.Config.Gossip.Key) - if err != nil { - return err - } - } - - // get the host portion of addr to use for binding - gossipHost := uri.Host() - gossipNodeSet, err := gossip.NewGossipNodeSet(uri.HostPort(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey) - if err != nil { - return err - } - m.Server.Cluster.NodeSet = gossipNodeSet - m.Server.Broadcaster = m.Server - m.Server.BroadcastReceiver = gossipNodeSet - m.Server.Gossiper = gossipNodeSet - case pilosa.ClusterStatic, pilosa.ClusterNone: - m.Server.Broadcaster = pilosa.NopBroadcaster - m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() - m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - m.Server.Gossiper = pilosa.NopGossiper - err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes) - if err != nil { - return err - } - default: - return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type) + diagnosticsInterval := time.Duration(0) + if m.Config.Metric.Diagnostics { + diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval) } - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) - return nil + statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + if err != nil { + return errors.Wrap(err, "new stats client") + } + + ln, err := getListener(*uri, TLSConfig) + if err != nil { + return errors.Wrap(err, "getting listener") + } + + c := GetHTTPClient(TLSConfig) + handler.API.RemoteClient = c + + m.Server, err = pilosa.NewServer( + pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), + pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), + pilosa.OptServerDataDir(m.Config.DataDir), + pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN), + pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest), + pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)), + pilosa.OptServerDiagnosticsInterval(diagnosticsInterval), + + pilosa.OptServerLogger(m.logger), + pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore), + pilosa.OptServerHandler(handler), + pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), + pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), + pilosa.OptServerStatsClient(statsClient), + pilosa.OptServerListener(ln), + pilosa.OptServerURI(uri), + pilosa.OptServerRemoteClient(c), + ) + + return errors.Wrap(err, "new server") } -// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path. -func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) { - // This is split out so it can be used in NewServeCmd as well as SetupServer - if path == "" { - return defaultWriter, nil - } else { - logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - return logFile, nil +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} + +// SetupNetworking sets up internode communication based on the configuration. +func (m *Command) SetupNetworking() error { + + m.Server.NodeID = m.Server.LoadNodeID() + + if m.Config.Cluster.Disabled { + m.Server.Cluster.Static = true + m.Server.Cluster.Coordinator = m.Server.NodeID + for _, address := range m.Config.Cluster.Hosts { + uri, err := pilosa.NewURIFromAddress(address) + if err != nil { + return errors.Wrap(err, "getting URI") + } + m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{ + URI: *uri, + }) + } + + m.Server.Broadcaster = pilosa.NopBroadcaster + m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes) + m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver + m.Server.Gossiper = pilosa.NopGossiper + return nil + } + + gossipPort, err := strconv.Atoi(m.Config.Gossip.Port) + if err != nil { + return errors.Wrap(err, "parsing port") + } + + // get the host portion of addr to use for binding + gossipHost := m.Server.URI.Host() + var transport *gossip.Transport + if m.GossipTransport != nil { + transport = m.GossipTransport + } else { + transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) + if err != nil { + return errors.Wrap(err, "getting transport") + } + } + + // Set Coordinator. + if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { + m.Server.Cluster.Coordinator = m.Server.NodeID + m.Server.Cluster.Node.IsCoordinator = true + } + + gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) + m.Server.Cluster.EventReceiver = gossipEventReceiver + gossipMemberSet, err := gossip.NewGossipMemberSet( + m.Server.NodeID, + m.Server.URI.Host(), + m.Config.Gossip, + gossipEventReceiver, + m.Server, + gossip.WithLogger(m.logger.Logger()), + gossip.WithTransport(transport), + ) + if err != nil { + return errors.Wrap(err, "getting memberset") + } + gossipMemberSet.Logger = m.logger + m.Server.Cluster.MemberSet = gossipMemberSet + m.Server.Broadcaster = m.Server + m.Server.BroadcastReceiver = gossipMemberSet + m.Server.Gossiper = gossipMemberSet + return nil } // Close shuts down the server. func (m *Command) Close() error { var logErr error serveErr := m.Server.Close() - logOutput := m.Server.LogOutput - if closer, ok := logOutput.(io.Closer); ok { + if closer, ok := m.logOutput.(io.Closer); ok { logErr = closer.Close() } close(m.Done) @@ -284,7 +343,30 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { return pilosa.NewExpvarStatsClient(), nil case "statsd": return statsd.NewStatsClient(host) - default: + case "nop", "none": return pilosa.NopStatsClient, nil + default: + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") } } + +// getListener gets a net.Listener based on the config. +func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { + // If bind URI has the https scheme, enable TLS + if uri.Scheme() == "https" && tlsconf != nil { + ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) + if err != nil { + return nil, errors.Wrap(err, "tls.Listener") + } + } else if uri.Scheme() == "http" { + // Open HTTP listener to determine port (if specified as :0). + ln, err = net.Listen("tcp", uri.HostPort()) + if err != nil { + return nil, errors.Wrap(err, "net.Listen") + } + } else { + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + } + + return ln, nil +} diff --git a/server/server_test.go b/server/server_test.go index 4281541da..512ce5c11 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -15,15 +15,11 @@ package server_test import ( - "bytes" "context" "encoding/json" "fmt" - "io" "io/ioutil" "math/rand" - "net/http" - "os" "reflect" "runtime" "sort" @@ -36,6 +32,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" ) // Ensure program can process queries and maintain consistency. @@ -45,24 +42,24 @@ func TestMain_Set_Quick(t *testing.T) { } if err := quick.Check(func(cmds []SetCommand) bool { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } // Execute SetBit() commands. for _, cmd := range cmds { - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && errors.Cause(err) != pilosa.ErrIndexExists { t.Fatal(err) } if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(rowID=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -78,7 +75,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -101,7 +98,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -121,7 +118,7 @@ func TestMain_Set_Quick(t *testing.T) { // Ensure program can set row attributes and retrieve them. func TestMain_SetRowAttrs(t *testing.T) { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create frames. @@ -137,36 +134,36 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="x", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(rowID=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -177,19 +174,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -198,7 +195,7 @@ func TestMain_SetRowAttrs(t *testing.T) { // Ensure program can set column attributes and retrieve them. func TestMain_SetColumnAttrs(t *testing.T) { - m := MustRunMain() + m := test.MustRunMain() defer m.Close() // Create frames. @@ -210,19 +207,19 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -233,54 +230,64 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } -// Ensure program can set column attributes with columnLabel option. -func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { - m := MustRunMain() - defer m.Close() +// Ensure inverse slices get handled correctly in a multi-node query. +func TestMain_InverseSlices(t *testing.T) { + mains := test.MustRunMainWithCluster(t, 2) + + m0 := mains[0] + m1 := mains[1] + + // Make sure to use node0 in the cluster. + var m *test.Main + if m0.Server.NodeID < m1.Server.NodeID { + m = m0 + } else { + m = m1 + } // Create frames. client := m.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal("create index:", err) + } + if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal("create frame:", err) } - // Set bits on row. - if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=100)`); err != nil { - t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=101)`); err != nil { - t.Fatal(err) + // Write data on cluster. + if _, err := m.Query("i", "", fmt.Sprintf(` + SetBit(col=1, frame="f", row=1000) + SetBit(col=1, frame="f", row=2000) + SetBit(col=1, frame="f", row=%d) + `, 1*pilosa.SliceWidth)); err != nil { + t.Fatal("setting bits:", err) } - // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { - t.Fatal(err) - } + time.Sleep(1 * time.Second) - // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { - t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + // Query the cluster. + if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { + t.Fatal("another bitmap query:", err) + } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { t.Fatalf("unexpected result: %s", res) } - } // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { - mains1 := NewMainArrayWithCluster(2) - m0 := mains1[0] + mains1 := test.MustRunMainWithCluster(t, 2) + m10 := mains1[0] + m11 := mains1[1] // Create frames. - client := m0.Client() + client := m10.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } @@ -289,47 +296,58 @@ func TestMain_FrameRestore(t *testing.T) { } // Write data on first cluster. - if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=100) - SetBit(rowID=1, frame="f", columnID=1000) - SetBit(rowID=1, frame="f", columnID=100000) - SetBit(rowID=1, frame="f", columnID=200000) - SetBit(rowID=1, frame="f", columnID=400000) - SetBit(rowID=1, frame="f", columnID=600000) - SetBit(rowID=1, frame="f", columnID=800000) + if _, err := m10.Query("i", "", ` + SetBit(row=1, frame="f", col=100) + SetBit(row=1, frame="f", col=1000) + SetBit(row=1, frame="f", col=100000) + SetBit(row=1, frame="f", col=200000) + SetBit(row=1, frame="f", col=400000) + SetBit(row=1, frame="f", col=600000) + SetBit(row=1, frame="f", col=800000) `); err != nil { t.Fatal("setting bits:", err) } // Query row on first cluster. - if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Start second cluster. - mains2 := NewMainArrayWithCluster(2) - m2 := mains2[0] - defer m2.Close() + mains2 := test.MustRunMainWithCluster(t, 2) + m20 := mains2[0] + defer m20.Close() + m21 := mains2[1] + defer m21.Close() // Import from first cluster. - client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } - if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil)) + if err != nil { + t.Fatal("new client:", err) + } + + if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create new index:", err) } - if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { t.Fatal("create new frame:", err) } - if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil { + + if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil { + t.Fatal("restore frame:", err) + } + if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil { t.Fatal("restore frame:", err) } // Query row on second cluster. - if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { + if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("2unexpected result: %s", res) @@ -378,226 +396,49 @@ func TestCountOpenFiles(t *testing.T) { } } -// Ensure program can send/receive broadcast messages. -func TestMain_SendReceiveMessage(t *testing.T) { - mains := NewMainArrayWithCluster(2) - m0 := mains[0] - defer m0.Close() +func TestMain_RecalculateHashes(t *testing.T) { + const clusterSize = 5 + cluster := test.MustRunMainWithCluster(t, clusterSize) - m1 := mains[1] - defer m1.Close() - - // Expected indexes and Frames - expected := map[string][]string{ - "i": []string{"f"}, - } - - // Create a client for each node. - client0 := m0.Client() - client1 := m1.Client() - - // Create indexes and frames on one node. + // Create the schema. + client0 := cluster[0].Client() if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) + t.Fatal("create index:", err) + } + if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil { + t.Fatal("create frame:", err) } - // Make sure node0 knows about the index and frame created. - schema0, err := client0.Schema(context.Background()) - if err != nil { - t.Fatal(err) - } - received0 := map[string][]string{} - for _, idx := range schema0 { - received0[idx.Name] = []string{} - for _, frame := range idx.Frames { - received0[idx.Name] = append(received0[idx.Name], frame.Name) + // Set some bits + data := []string{} + for rowID := 1; rowID < 10; rowID++ { + for columnID := 1; columnID < 100; columnID++ { + data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) } } - if !reflect.DeepEqual(received0, expected) { - t.Fatalf("unexpected schema on node0: %s", received0) + if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { + t.Fatal("setting bits:", err) } - // Make sure node1 knows about the index and frame created. - schema1, err := client1.Schema(context.Background()) + // Calculate caches on the first node + err := cluster[0].RecalculateCaches() if err != nil { - t.Fatal(err) + t.Fatalf("recalculating caches: %v", err) } - received1 := map[string][]string{} - for _, idx := range schema1 { - received1[idx.Name] = []string{} - for _, frame := range idx.Frames { - received1[idx.Name] = append(received1[idx.Name], frame.Name) + + target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}` + + // Run a TopN query on all nodes. The result should be the same as the target. + for _, m := range cluster { + res, err := m.Query("i", "", `TopN(frame="f")`) + if err != nil { + t.Fatal(err) + } + res = strings.TrimSpace(res) + if sortedString(target) != sortedString(res) { + t.Fatalf("%v != %v", target, res) } } - if !reflect.DeepEqual(received1, expected) { - t.Fatalf("unexpected schema on node1: %s", received1) - } - - // Write data on first node. - if _, err := m0.Query("i", "", ` - SetBit(rowID=1, frame="f", columnID=1) - SetBit(rowID=1, frame="f", columnID=2400000) - `); err != nil { - t.Fatal(err) - } - - // We have to wait for the broadcast message to be sent before checking state. - time.Sleep(1 * time.Second) - - // Make sure node0 knows about the latest MaxSlice. - maxSlices0, err := client0.MaxSliceByIndex(context.Background()) - if err != nil { - t.Fatal(err) - } - if maxSlices0["i"] != 2 { - t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) - } - - // Make sure node1 knows about the latest MaxSlice. - maxSlices1, err := client1.MaxSliceByIndex(context.Background()) - if err != nil { - t.Fatal(err) - } - if maxSlices1["i"] != 2 { - t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) - } - - // Write input definition to the first node. - if _, err := m0.CreateDefinition("i", "test", `{ - "frames": [{"name": "event-time", - "options": { - "cacheType": "ranked", - "timeQuantum": "YMD" - }}], - "fields": [{"name": "columnID", - "primaryKey": true - }]} - `); err != nil { - t.Fatal(err) - } - - // We have to wait for the broadcast message to be sent before checking state. - time.Sleep(1 * time.Second) - - frame0 := m0.Server.Holder.Frame("i", "event-time") - if frame0 == nil { - t.Fatal("frame not found") - } - frame1 := m1.Server.Holder.Frame("i", "event-time") - if frame1 == nil { - t.Fatal("frame not found") - } -} - -// Main represents a test wrapper for main.Main. -type Main struct { - *server.Command - - Stdin bytes.Buffer - Stdout bytes.Buffer - Stderr bytes.Buffer -} - -// NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain() *Main { - path, err := ioutil.TempDir("", "pilosa-") - if err != nil { - panic(err) - } - - m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} - m.Server.Network = *test.Network - m.Config.DataDir = path - m.Config.Bind = "localhost:0" - m.Config.Cluster.Type = "static" - m.Command.Stdin = &m.Stdin - m.Command.Stdout = &m.Stdout - m.Command.Stderr = &m.Stderr - - if testing.Verbose() { - m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) - m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) - } - - return m -} - -func NewMainArrayWithCluster(size int) []*Main { - cluster, err := test.NewServerCluster(size) - if err != nil { - panic(err) - } - mainArray := make([]*Main, size) - for i := 0; i < size; i++ { - mainArray[i] = &Main{Command: cluster.Servers[i]} - } - return mainArray -} - -// MustRunMain returns a new, running Main. Panic on error. -func MustRunMain() *Main { - m := NewMain() - if err := m.Run(); err != nil { - panic(err) - } - return m -} - -// Close closes the program and removes the underlying data directory. -func (m *Main) Close() error { - defer os.RemoveAll(m.Config.DataDir) - return m.Command.Close() -} - -// Reopen closes the program and reopens it. -func (m *Main) Reopen() error { - if err := m.Command.Close(); err != nil { - return err - } - - // Create new main with the same config. - config := m.Config - m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) - m.Server.Network = *test.Network - m.Config = config - - // Run new program. - if err := m.Run(); err != nil { - return err - } - return nil -} - -// URL returns the base URL string for accessing the running program. -func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } - -// Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) - if err != nil { - panic(err) - } - return client -} - -// Query executes a query against the program through the HTTP API. -func (m *Main) Query(index, rawQuery, query string) (string, error) { - resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - return resp.Body, nil -} - -// CreateDefinition. -func (m *Main) CreateDefinition(index, def, query string) (string, error) { - resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - return resp.Body, nil } // SetCommand represents a command to set a bit. @@ -650,38 +491,12 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { } // ParseConfig parses s into a Config. -func ParseConfig(s string) (pilosa.Config, error) { - var c pilosa.Config +func ParseConfig(s string) (server.Config, error) { + var c server.Config _, err := toml.Decode(s, &c) return c, err } -// MustDo executes http.Do() with an http.NewRequest(). Panic on error. -func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) - if err != nil { - panic(err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err) - } - - return &httpResponse{Response: resp, Body: string(buf)} -} - -// httpResponse is a wrapper for http.Response that holds the Body as a string. -type httpResponse struct { - *http.Response - Body string -} - // MustMarshalJSON marshals v into a string. Panic on error. func MustMarshalJSON(v interface{}) string { buf, err := json.Marshal(v) @@ -691,6 +506,12 @@ func MustMarshalJSON(v interface{}) string { return string(buf) } +func sortedString(s string) string { + arr := strings.Split(s, "") + sort.Strings(arr) + return strings.Join(arr, "") +} + // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..9e10374d7 --- /dev/null +++ b/server_test.go @@ -0,0 +1,35 @@ +package pilosa_test + +import ( + "context" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// TestMonitorAntiEntropy is a regression test which which caught a bug where +// pilosa.Server was not having its remoteClient field set by an option and so +// it was using a nil client in monitorAntiEntropy. +func TestMonitorAntiEntropy(t *testing.T) { + cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1)) + client := cluster[1].Client() + err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) + if err != nil { + t.Fatalf("creating frame: %v", err) + } + + time.Sleep(time.Millisecond * 2) + for _, m := range cluster { + err := m.Close() + if err != nil { + t.Fatal(err) + } + } + +} diff --git a/statik/.gitignore b/statik/.gitignore index 514ee40a1..485c0c57d 100644 --- a/statik/.gitignore +++ b/statik/.gitignore @@ -1 +1 @@ -statik.go +/statik.go diff --git a/statik/doc.go b/statik/doc.go deleted file mode 100644 index 85edd9e6f..000000000 --- a/statik/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package statik contains static assets for the Web UI. `go generate` will -// produce statik.go, which is ignored by git. -package statik diff --git a/roaring/assembly_generic.go b/statik/filesystem.go similarity index 50% rename from roaring/assembly_generic.go rename to statik/filesystem.go index 997bfdaa3..e3bf95cb1 100644 --- a/roaring/assembly_generic.go +++ b/statik/filesystem.go @@ -11,17 +11,27 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +//go:generate statik -src=../webui -dest=.. +// +// Package statik contains static assets for the Web UI. `go generate` or +// `make generate-statik` will produce statik.go, which is ignored by git. +package statik -// +build !amd64 +import ( + "net/http" -package roaring + "github.com/pilosa/pilosa" + "github.com/rakyll/statik/fs" +) -func hasAsm() bool {return false} +// Ensure nopFileSystem implements interface. +var _ pilosa.FileSystem = &FileSystem{} +// FileSystem represents a static FileSystem. +type FileSystem struct{} -func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) } -func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) } -func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) } -func popcntOrSlice(s, m []uint64) uint64 { return popcntOrSliceGo(s, m) } -func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) } -func popcnt(s uint64) uint64 { return popcntGo(s) } +// New is a statik implementation of FileSystem New method. +func (s *FileSystem) New() (http.FileSystem, error) { + return fs.New() +} diff --git a/stats.go b/stats.go index 998b84511..313708fbc 100644 --- a/stats.go +++ b/stats.go @@ -16,7 +16,6 @@ package pilosa import ( "expvar" - "io" "sort" "strings" "sync" @@ -57,7 +56,7 @@ type StatsClient interface { Timing(name string, value time.Duration, rate float64) // SetLogger Set the logger output type - SetLogger(logger io.Writer) + SetLogger(logger Logger) // Starts the service Open() @@ -79,7 +78,7 @@ func (c *nopStatsClient) Gauge(name string, value float64, rate float64) func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {} func (c *nopStatsClient) Set(name string, value string, rate float64) {} func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} -func (c *nopStatsClient) SetLogger(logger io.Writer) {} +func (c *nopStatsClient) SetLogger(logger Logger) {} func (c *nopStatsClient) Open() {} func (c *nopStatsClient) Close() error { return nil } @@ -154,7 +153,7 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 } // SetLogger has no logger. -func (c *ExpvarStatsClient) SetLogger(logger io.Writer) { +func (c *ExpvarStatsClient) SetLogger(logger Logger) { } // Open no-op. @@ -226,7 +225,7 @@ func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) } // SetLogger Sets the StatsD logger output type. -func (a MultiStatsClient) SetLogger(logger io.Writer) { +func (a MultiStatsClient) SetLogger(logger Logger) { for _, c := range a { c.SetLogger(logger) } diff --git a/stats_test.go b/stats_test.go index 93d7d9bb1..d6c748c5f 100644 --- a/stats_test.go +++ b/stats_test.go @@ -16,8 +16,6 @@ package pilosa_test import ( "context" - "io" - "io/ioutil" "net/http" "strings" "testing" @@ -38,7 +36,6 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.Stats.SetLogger(ioutil.Discard) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -143,7 +140,7 @@ func TestStatsCount_Bitmap(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -174,7 +171,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -206,7 +203,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { return }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -218,10 +215,10 @@ func TestStatsCount_CreateIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -242,7 +239,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -250,7 +247,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "deleteIndex" { t.Errorf("Expected deleteIndex, Results %s", name) @@ -271,7 +268,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -279,7 +276,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "createFrame" { t.Errorf("Expected createFrame, Results %s", name) @@ -303,7 +300,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false // Create index. @@ -311,7 +308,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "deleteFrame" { t.Errorf("Expected deleteFrame, Results %s", name) @@ -357,6 +354,6 @@ func (c *MockStats) Gauge(name string, value float64, rate float64) {} func (c *MockStats) Histogram(name string, value float64, rate float64) {} func (c *MockStats) Set(name string, value string, rate float64) {} func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} -func (c *MockStats) SetLogger(logger io.Writer) {} +func (c *MockStats) SetLogger(logger pilosa.Logger) {} func (c *MockStats) Open() {} func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go index d46dd637f..0cec63718 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,16 +15,13 @@ package statsd import ( - "io" - "io/ioutil" - "log" "time" "github.com/DataDog/datadog-go/statsd" "github.com/pilosa/pilosa" ) -// StatsD protocal wrapper using the DataDog library that added Tags to the StatsD protocal +// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol // statsD defailt host is "127.0.0.1:8125" const ( @@ -40,9 +37,9 @@ var _ pilosa.StatsClient = &StatsClient{} // StatsClient represents a StatsD implementation of pilosa.StatsClient. type StatsClient struct { - client *statsd.Client - tags []string - logOutput io.Writer + client *statsd.Client + tags []string + logger pilosa.Logger } // NewStatsClient returns a new instance of StatsClient. @@ -53,8 +50,8 @@ func NewStatsClient(host string) (*StatsClient, error) { } return &StatsClient{ - client: c, - logOutput: ioutil.Discard, + client: c, + logger: pilosa.NopLogger, }, nil } @@ -74,16 +71,16 @@ func (c *StatsClient) Tags() []string { // WithTags returns a new client with additional tags appended. func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ - client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), - logOutput: c.logOutput, + client: c.client, + tags: pilosa.UnionStringSlice(c.tags, tags), + logger: c.logger, } } // Count tracks the number of times something occurs per second. func (c *StatsClient) Count(name string, value int64, rate float64) { if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } @@ -91,44 +88,39 @@ func (c *StatsClient) Count(name string, value int64, rate float64) { func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) if err := c.client.Count(Prefix+name, value, tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Count error: %s", err) + c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // Gauge sets the value of a metric. func (c *StatsClient) Gauge(name string, value float64, rate float64) { if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Gauge error: %s", err) + c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. func (c *StatsClient) Histogram(name string, value float64, rate float64) { if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Histogram error: %s", err) + c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. func (c *StatsClient) Set(name string, value string, rate float64) { if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Set error: %s", err) + c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil { - c.logger().Printf("statsd.StatsClient.Timing error: %s", err) + c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } -// SetLogger has no logger -func (c *StatsClient) SetLogger(logger io.Writer) { - c.logOutput = logger -} - -// logger returns a logger that writes to LogOutput -func (c *StatsClient) logger() *log.Logger { - return log.New(c.logOutput, "", log.LstdFlags) +// SetLogger sets the logger for client. +func (c *StatsClient) SetLogger(logger pilosa.Logger) { + c.logger = logger } diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go index 4d5c1cc98..38228349b 100644 --- a/statsd/statsd_test.go +++ b/statsd/statsd_test.go @@ -15,7 +15,6 @@ package statsd_test import ( - "io/ioutil" "reflect" "testing" "time" @@ -31,7 +30,6 @@ func TestStatsClient_WithTags(t *testing.T) { t.Fatal(err) } defer c.Close() - c.SetLogger(ioutil.Discard) // Create a new client with additional tags. c1 := c.WithTags("foo", "bar") diff --git a/test/attr.go b/test/attr.go index 9363011b1..16e8ff334 100644 --- a/test/attr.go +++ b/test/attr.go @@ -22,15 +22,16 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // AttrStore represents a test wrapper for pilosa.AttrStore. type AttrStore struct { - *pilosa.AttrStore + pilosa.AttrStore } // NewAttrStore returns a new instance of AttrStore. -func NewAttrStore() *AttrStore { +func NewAttrStore(string) pilosa.AttrStore { f, err := ioutil.TempFile("", "pilosa-attr-") if err != nil { panic(err) @@ -38,7 +39,7 @@ func NewAttrStore() *AttrStore { f.Close() os.Remove(f.Name()) - return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())} + return &AttrStore{boltdb.NewAttrStore(f.Name())} } func BenchmarkAttrStore_Duplicate(b *testing.B) { @@ -74,8 +75,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) { } // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() *AttrStore { - s := NewAttrStore() +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") if err := s.Open(); err != nil { panic(err) } diff --git a/test/cluster.go b/test/cluster.go index adf82fc0a..77cf496a8 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -15,24 +15,43 @@ package test import ( + "bufio" + "bytes" "fmt" + "io/ioutil" + "path/filepath" + "sync" + "time" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { + path, err := ioutil.TempDir("", "pilosa-cluster-") + if err != nil { + panic(err) + } + c := pilosa.NewCluster() c.ReplicaN = 1 c.Hasher = NewModHasher() + c.Path = path + c.Topology = pilosa.NewTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ - Scheme: "http", - Host: fmt.Sprintf("host%d", i), + ID: fmt.Sprintf("node%d", i), + URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } + c.Node = c.Nodes[0] + c.Coordinator = c.Nodes[0].ID + c.SetState(pilosa.ClusterStateNormal) + return c } @@ -53,3 +72,379 @@ type ConstHasher struct { func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } + +// NewURI is a test URI creator that intentionally swallows errors. +func NewURI(scheme, host string, port uint16) pilosa.URI { + uri := pilosa.DefaultURI() + uri.SetScheme(scheme) + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +func NewURIFromHostPort(host string, port uint16) pilosa.URI { + uri := pilosa.DefaultURI() + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +// TestCluster represents a cluster of test nodes, each of which +// has a pilosa.Cluster. +type TestCluster struct { + Clusters []*pilosa.Cluster + + common *commonClusterSettings + + mu sync.RWMutex + resizing bool + resizeDone chan struct{} +} + +type commonClusterSettings struct { + Nodes []*pilosa.Node +} + +func (t *TestCluster) CreateIndex(name string) error { + for _, c := range t.Clusters { + if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { + return err + } + } + return nil +} + +func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error { + for _, c := range t.Clusters { + idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) + if err != nil { + return err + } + if _, err := idx.CreateFrame(frame, opt); err != nil { + return err + } + } + return nil +} +func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { + // Determine which node should receive the SetBit. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := colID / pilosa.SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetBit(view, rowID, colID, x) + if err != nil { + return err + } + } + + return nil +} + +func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { + // Determine which node should receive the SetFieldValue. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := columnID / pilosa.SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetFieldValue(columnID, name, value) + if err != nil { + return err + } + } + + return nil +} + +func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { + for _, c := range t.Clusters { + if c.Node.ID == id { + return c + } + } + return nil +} + +// AddNode adds a node to the cluster and (potentially) starts a resize job. +func (t *TestCluster) AddNode(saveTopology bool) error { + id := len(t.Clusters) + + c, err := t.addCluster(id, saveTopology) + if err != nil { + return err + } + + // Send NodeJoin event to coordinator. + if id > 0 { + coord := t.Clusters[0] + ev := &pilosa.NodeEvent{ + Event: pilosa.NodeJoin, + Node: c.Node, + } + + if err := coord.ReceiveEvent(ev); err != nil { + return err + } + + // Wait for the AddNode job to finish. + if c.State() != pilosa.ClusterStateNormal { + t.resizeDone = make(chan struct{}) + t.mu.Lock() + t.resizing = true + t.mu.Unlock() + <-t.resizeDone + } + } + + return nil +} + +// WriteTopology writes the given topology to disk. +func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { + if buf, err := proto.Marshal(top.Encode()); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { + + id := fmt.Sprintf("node%d", i) + uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) + + node := &pilosa.Node{ + ID: id, + URI: uri, + } + + // add URI to common + //t.common.NodeIDs = append(t.common.NodeIDs, id) + //sort.Sort(t.common.NodeIDs) + + // add node to common + t.common.Nodes = append(t.common.Nodes, node) + + // create node-specific temp directory + path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + if err != nil { + return nil, err + } + + // holder + h := pilosa.NewHolder() + h.Path = path + + // cluster + c := pilosa.NewCluster() + c.ReplicaN = 1 + c.Hasher = NewModHasher() + c.Path = path + c.Topology = pilosa.NewTopology() + c.Holder = h + c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) + c.Node = node + c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator + c.Broadcaster = t + + // add nodes + if saveTopology { + for _, n := range t.common.Nodes { + c.AddNode(n) + } + } + + // Add this node to the TestCluster. + t.Clusters = append(t.Clusters, c) + + return c, nil +} + +// NewTestCluster returns a new instance of test.Cluster. +func NewTestCluster(n int) *TestCluster { + + tc := &TestCluster{ + common: &commonClusterSettings{}, + } + + // add clusters + for i := 0; i < n; i++ { + _, err := tc.addCluster(i, true) + if err != nil { + panic(err) + } + } + return tc +} + +// SetState sets the state of the cluster on each node. +func (t *TestCluster) SetState(state string) { + for _, c := range t.Clusters { + c.SetState(state) + } +} + +// Open opens all clusters in the test cluster. +func (t *TestCluster) Open() error { + for _, c := range t.Clusters { + if err := c.Open(); err != nil { + return err + } + if err := c.Holder.Open(); err != nil { + return err + } + if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { + return err + } + } + + // Start the listener on the coordinator. + if len(t.Clusters) == 0 { + return nil + } + t.Clusters[0].ListenForJoins() + + return nil +} + +// Close closes all clusters in the test cluster. +func (t *TestCluster) Close() error { + for _, c := range t.Clusters { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + +// TestCluster implements Broadcaster interface. + +// SendSync is a test implemenetation of Broadcaster SendSync method. +func (t *TestCluster) SendSync(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ClusterStatus: + // Apply the send message to all nodes (except the coordinator). + for _, c := range t.Clusters { + c.MergeClusterStatus(obj) + } + t.mu.RLock() + if obj.State == pilosa.ClusterStateNormal && t.resizing { + close(t.resizeDone) + } + t.mu.RUnlock() + } + + return nil +} + +// SendAsync is a test implemenetation of Broadcaster SendAsync method. +func (t *TestCluster) SendAsync(pb proto.Message) error { + return nil +} + +// SendTo is a test implemenetation of Broadcaster SendTo method. +func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ResizeInstruction: + err := t.FollowResizeInstruction(obj) + if err != nil { + return err + } + case *internal.ResizeInstructionComplete: + coord := t.clusterByID(to.ID) + go coord.MarkResizeInstructionComplete(obj) + } + return nil +} + +// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Node: instr.Node, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrNode := pilosa.DecodeNode(instr.Node) + destCluster := t.clusterByID(instrNode.ID) + + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } + + for _, src := range instr.Sources { + srcNode := pilosa.DecodeNode(src.Node) + srcCluster := t.clusterByID(srcNode.ID) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { + return err + } + } + + return nil + }(); err != nil { + complete.Error = err.Error() + } + + node := pilosa.DecodeNode(instr.Coordinator) + if err := t.SendTo(node, complete); err != nil { + return err + } + + return nil +} diff --git a/test/executor.go b/test/executor.go index 9b4233035..8cd6391c3 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -30,18 +31,17 @@ type Executor struct { var remoteClient *http.Client func init() { - remoteClient = pilosa.GetHTTPClient(nil) + remoteClient = server.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. -// The executor always matches the hostname of the first cluster node. +// The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { executor := pilosa.NewExecutor(remoteClient) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster - e.Scheme = cluster.Nodes[0].Scheme - e.Host = cluster.Nodes[0].Host + e.Node = cluster.Nodes[0] return e } diff --git a/test/fragment.go b/test/fragment.go index b208e7a3a..39caa8db2 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -27,7 +27,7 @@ const SliceWidth = pilosa.SliceWidth // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { *pilosa.Fragment - RowAttrStore *AttrStore + RowAttrStore pilosa.AttrStore } // NewFragment returns a new instance of Fragment with a temporary path. @@ -43,7 +43,7 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra RowAttrStore: MustOpenAttrStore(), } f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore return f } @@ -78,7 +78,7 @@ func (f *Fragment) Reopen() error { f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore if err := f.Open(); err != nil { return err } diff --git a/test/handler.go b/test/handler.go index 4743ea30c..50b081812 100644 --- a/test/handler.go +++ b/test/handler.go @@ -17,7 +17,6 @@ package test import ( "context" "encoding/json" - "errors" "io" "io/ioutil" "net/http" @@ -41,11 +40,12 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.Handler.Executor = &h.Executor - h.Handler.LogOutput = ioutil.Discard + h.API = pilosa.NewAPI() + h.Handler.API = h.API + h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster + h.API.Broadcaster = pilosa.NopBroadcaster return h } @@ -75,48 +75,33 @@ func NewServer() *Server { } s.Server = httptest.NewServer(s.Handler.Handler) - // Update handler to use hostname. - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - panic(err) - } - s.Handler.URI = uri - // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster + s.Handler.API.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.API.Cluster = NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() return s } -// LocalStatus returns the state of the local node as well as the -// holder (indexes/frames) according to the local node. +// LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.URI.HostPort(), - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.URI.HostPort()) - } - - return &ns, nil + return nil, nil } -// ClusterStatus returns the NodeState for all nodes in the cluster. +// ClusterStatus exists so that test.Server implements StatusHandler. func (s *Server) ClusterStatus() (proto.Message, error) { - // Assuming we are only testing this with one Node - // So just return its status - return s.LocalStatus() + id := "test-node" + uri := pilosa.DefaultURI() + node := &pilosa.Node{ + ID: id, + URI: *uri, + } + return &internal.ClusterStatus{ + ClusterID: "", + State: pilosa.ClusterStateNormal, + Nodes: pilosa.EncodeNodes([]*pilosa.Node{node}), + }, nil } // HandleRemoteStatus just need to implement a nop to complete the Interface @@ -125,12 +110,12 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } // Host returns the hostname of the running server. func (s *Server) Host() string { return MustParseURLHost(s.URL) } -func (s *Server) HostURI() *pilosa.URI { +func (s *Server) HostURI() pilosa.URI { uri, err := pilosa.NewURIFromAddress(s.URL) if err != nil { panic(err) } - return uri + return *uri } // MustParseURLHost parses rawurl and returns the hostname. Panic on error. diff --git a/test/holder.go b/test/holder.go index c9d21d2f2..97b399bea 100644 --- a/test/holder.go +++ b/test/holder.go @@ -15,17 +15,16 @@ package test import ( - "bytes" "io/ioutil" "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Holder is a test wrapper for pilosa.Holder. type Holder struct { *pilosa.Holder - LogOutput bytes.Buffer } // NewHolder returns a new instance of Holder with a temporary path. @@ -37,7 +36,7 @@ func NewHolder() *Holder { h := &Holder{Holder: pilosa.NewHolder()} h.Path = path - h.Holder.LogOutput = &h.LogOutput + h.Holder.NewAttrStore = boltdb.NewAttrStore return h } @@ -60,10 +59,11 @@ func (h *Holder) Close() error { // Reopen instantiates and opens a new holder. // Note that the holder must be Closed first. func (h *Holder) Reopen() error { - path, logOutput := h.Path, h.Holder.LogOutput + path, logger := h.Path, h.Holder.Logger h.Holder = pilosa.NewHolder() h.Holder.Path = path - h.Holder.LogOutput = logOutput + h.Holder.Logger = logger + h.Holder.NewAttrStore = boltdb.NewAttrStore if err := h.Holder.Open(); err != nil { return err } diff --git a/test/logger.go b/test/logger.go new file mode 100644 index 000000000..b4a0079b1 --- /dev/null +++ b/test/logger.go @@ -0,0 +1,48 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +// BufferLogger represents a test Logger that holds log messages +// in a buffer for review. +type BufferLogger struct { + buf *bytes.Buffer +} + +// NewBufferLogger returns a new instance of BufferLogger. +func NewBufferLogger() *BufferLogger { + return &BufferLogger{ + buf: &bytes.Buffer{}, + } +} + +func (b *BufferLogger) Printf(format string, v ...interface{}) { + s := fmt.Sprintf(format, v...) + _, err := b.buf.WriteString(s) + if err != nil { + panic(err) + } +} + +func (b *BufferLogger) Debugf(format string, v ...interface{}) {} + +func (b *BufferLogger) ReadAll() ([]byte, error) { + return ioutil.ReadAll(b.buf) +} diff --git a/test/pilosa.go b/test/pilosa.go index e14b2d032..f6e3a3457 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -16,119 +16,286 @@ package test import ( "bytes" + "fmt" + "io" "io/ioutil" - "net" - "strconv" + "net/http" + "os" + "strings" "testing" + "time" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) -func MustNewRunningServer(t *testing.T) *server.Command { - s, err := newServer() - if err != nil { - t.Fatalf("getting new server: %v", err) - } +//////////////////////////////////////////////////////////////////////////////////// +// Main represents a test wrapper for main.Main. +type Main struct { + *server.Command - err = s.Run() - if err != nil { - t.Fatalf("running new pilosa server: %v", err) - } - return s + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer } -func newServer() (*server.Command, error) { - s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard) +type MainOpt func(m *Main) error - port, err := findPort() - if err != nil { - return nil, errors.Wrap(err, "getting port") +func OptAntiEntropyInterval(dur time.Duration) MainOpt { + return func(m *Main) error { + m.Command.Config.AntiEntropy.Interval = toml.Duration(dur) + return nil } - s.Config.Bind = "localhost:" + strconv.Itoa(port) - - gport, err := findPort() - if err != nil { - return nil, errors.Wrap(err, "getting gossip port") - } - s.Config.GossipPort = strconv.Itoa(gport) - - s.Config.GossipSeed = "localhost:" + s.Config.GossipPort - s.Config.Cluster.Type = "gossip" - td, err := ioutil.TempDir("", "") - if err != nil { - return nil, errors.Wrap(err, "temp dir") - } - s.Config.DataDir = td - return s, nil } -func findPort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", ":0") +// NewMain returns a new instance of Main with a temporary data directory and random port. +func NewMain(opts ...MainOpt) *Main { + path, err := ioutil.TempDir("", "pilosa-") if err != nil { - return 0, errors.Wrap(err, "resolving new port addr") + panic(err) } - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, errors.Wrap(err, "listening to get new port") - } - port := l.Addr().(*net.TCPAddr).Port - err = l.Close() - if err != nil { - return port, errors.Wrap(err, "closing listener") - } - return port, nil + m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} + m.Config.DataDir = path + m.Config.Bind = "http://localhost:0" + m.Config.Cluster.Disabled = true + m.Command.Stdin = &m.Stdin + m.Command.Stdout = &m.Stdout + m.Command.Stderr = &m.Stderr + for _, opt := range opts { + err := opt(m) + if err != nil { + panic(err) + } + + } + err = m.SetupServer() + if err != nil { + panic(err) + } + + if testing.Verbose() { + m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) + m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) + } + + return m } -func MustFindPort(t *testing.T) int { - port, err := findPort() +// NewMainWithCluster returns a new instance of Main with clustering enabled. +func NewMainWithCluster(isCoordinator bool, opts ...MainOpt) *Main { + m := NewMain(opts...) + m.Config.Cluster.Disabled = false + m.Config.Cluster.Coordinator = isCoordinator + return m +} + +// MustRunMainWithCluster ruturns a running array of *Main where +// all nodes are joined via memberlist (i.e. clustering enabled). +func MustRunMainWithCluster(t *testing.T, size int, opts ...MainOpt) []*Main { + ma, err := runMainWithCluster(size, opts...) if err != nil { - t.Fatalf("allocating new port: %v", err) + t.Fatalf("new main array with cluster: %v", err) } - return port + return ma } -type Cluster struct { - Servers []*server.Command -} - -func MustNewServerCluster(t *testing.T, size int) *Cluster { - cluster, err := NewServerCluster(size) - if err != nil { - t.Fatalf("new cluster: %v", err) +// runMainWithCluster runs an array of *Main where all nodes are +// joined via memberlist (i.e. clustering enabled). +func runMainWithCluster(size int, opts ...MainOpt) ([]*Main, error) { + if size == 0 { + return nil, errors.New("cluster must contain at least one node") } - return cluster -} -func NewServerCluster(size int) (cluster *Cluster, err error) { - cluster = &Cluster{ - Servers: make([]*server.Command, size), - } - hosts := make([]string, size) + mains := make([]*Main, size) + + gossipHost := "localhost" + gossipPort := 0 + var err error + var gossipSeeds = make([]string, size) + for i := 0; i < size; i++ { - s, err := newServer() + m := NewMainWithCluster(i == 0, opts...) + m.Config.Cluster.Disabled = false + + gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) if err != nil { - return nil, errors.Wrap(err, "new server") + return nil, errors.Wrap(err, "RunWithTransport") } - cluster.Servers[i] = s - hosts[i] = s.Config.Bind - s.Config.GossipSeed = cluster.Servers[0].Config.GossipSeed + mains[i] = m } - for _, s := range cluster.Servers { - s.Config.Cluster.Hosts = hosts - } - for i, s := range cluster.Servers { - err := s.Run() - if err != nil { - for j := 0; j <= i; j++ { - cluster.Servers[j].Close() - } - return nil, errors.Wrapf(err, "starting server %d of %d. Config: %#v", i+1, size, s.Config) - } - } - - return cluster, nil + return mains, nil +} + +// MustRunMain returns a new, running Main. Panic on error. +func MustRunMain() *Main { + m := NewMain() + m.Config.Metric.Diagnostics = false // Disable diagnostics. + if err := m.Start(); err != nil { + panic(err) + } + return m +} + +// Close closes the program and removes the underlying data directory. +func (m *Main) Close() error { + defer os.RemoveAll(m.Config.DataDir) + return m.Command.Close() +} + +// Reopen closes the program and reopens it. +func (m *Main) Reopen() error { + if err := m.Command.Close(); err != nil { + return err + } + + // Create new main with the same config. + config := m.Command.Config + m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) + m.Command.Config = config + err := m.SetupServer() + if err != nil { + return errors.Wrap(err, "setting up server") + } + + m.Server.NewAttrStore = boltdb.NewAttrStore + m.Server.Holder.NewAttrStore = m.Server.NewAttrStore + + // Run new program. + if err := m.Start(); err != nil { + return err + } + return nil +} + +// RunWithTransport runs Main and returns the dynamically allocated gossip port. +func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (seed string, err error) { + defer close(m.Started) + + /* + TEST: + - SetupServer (just static settings from config) + - OpenListener (sets Server.Name to use in gossip) + - NewTransport (gossip) + - SetupNetworking (does the gossip or static stuff) - uses Server.Name + - Open server + + PRODUCTION: + - SetupServer (just static settings from config) + - SetupNetworking (does the gossip or static stuff) - calls NewTransport + - Open server - calls OpenListener + */ + + // SetupServer + err = m.SetupServer() + if err != nil { + return seed, err + } + + // Open gossip transport to use in SetupServer. + transport, err := gossip.NewTransport(host, bindPort, nil) + if err != nil { + return seed, err + } + m.GossipTransport = transport + + if len(joinSeeds) != 0 { + m.Config.Gossip.Seeds = joinSeeds + } else { + m.Config.Gossip.Seeds = []string{transport.URI.String()} + } + + seed = transport.URI.String() + + // SetupNetworking + err = m.SetupNetworking() + if err != nil { + return seed, err + } + + if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { + return seed, err + } + + m.Server.Cluster.Static = false + + // Initialize server. + err = m.Server.Open() + if err != nil { + return seed, err + } + + return seed, nil +} + +// URL returns the base URL string for accessing the running program. +func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } + +// Client returns a client to connect to the program. +func (m *Main) Client() *pilosa.InternalHTTPClient { + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) + if err != nil { + panic(err) + } + return client +} + +// Query executes a query against the program through the HTTP API. +func (m *Main) Query(index, rawQuery, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + +func (m *Main) RecalculateCaches() error { + resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") + if resp.StatusCode != 204 { + return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return nil +} + +//////////////////////////////////////////////////////////////////////////////////// + +// MustDo executes http.Do() with an http.NewRequest(). Panic on error. +func MustDo(method, urlStr string, body string) *httpResponse { + req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) + if err != nil { + panic(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + + return &httpResponse{Response: resp, Body: string(buf)} +} + +// httpResponse is a wrapper for http.Response that holds the Body as a string. +type httpResponse struct { + *http.Response + Body string } diff --git a/test/pilosa_test.go b/test/pilosa_test.go index ca2104866..583a244c0 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -15,28 +15,35 @@ package test_test import ( + "encoding/json" "net/http" "testing" - "encoding/json" - + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" ) func TestNewCluster(t *testing.T) { - cluster := test.MustNewServerCluster(t, 3) - response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status") + numNodes := 3 + cluster := test.MustRunMainWithCluster(t, numNodes) + coordinator := cluster[0].Server.Cluster.Coordinator + for i := 1; i < numNodes; i++ { + if coordi := cluster[i].Server.Cluster.Coordinator; coordi != coordinator { + t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator) + } + } + + response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status") if err != nil { t.Fatalf("getting schema: %v", err) } dec := json.NewDecoder(response.Body) body := struct { - Status struct { - Nodes []struct { - Host string - Schema string - State string - } + State string + Nodes []struct { + Scheme string + Host string + Port int } }{} @@ -50,13 +57,11 @@ func TestNewCluster(t *testing.T) { t.Fatalf("encoding: %v", err) } - if len(body.Status.Nodes) != 3 { + if len(body.Nodes) != 3 { t.Fatalf("wrong number of nodes in status: %s", bytes) } - for i, node := range body.Status.Nodes { - if node.State != "UP" { - t.Fatalf("node %d should be up but is %s", i, node.State) - } + if body.State != pilosa.ClusterStateNormal { + t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State) } } diff --git a/test/test.go b/test/test.go index c83b657c0..01f980ad2 100644 --- a/test/test.go +++ b/test/test.go @@ -13,10 +13,3 @@ // limitations under the License. package test - -import "flag" - -// Test flags. -var ( - Network = flag.String("network", "tcp", "network name") -) diff --git a/toml/toml.go b/toml/toml.go new file mode 100644 index 000000000..5193ad787 --- /dev/null +++ b/toml/toml.go @@ -0,0 +1,30 @@ +package toml + +import "time" + +// Duration is a TOML wrapper type for time.Duration. +type Duration time.Duration + +// String returns the string representation of the duration. +func (d Duration) String() string { return time.Duration(d).String() } + +// UnmarshalText parses a TOML value into a duration value. +func (d *Duration) UnmarshalText(text []byte) error { + v, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + + *d = Duration(v) + return nil +} + +// MarshalText writes duration value in text format. +func (d Duration) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// MarshalTOML write duration into valid TOML. +func (d Duration) MarshalTOML() ([]byte, error) { + return []byte(d.String()), nil +} diff --git a/uri.go b/uri.go index 21028fe44..beb5060b5 100644 --- a/uri.go +++ b/uri.go @@ -15,11 +15,14 @@ package pilosa import ( + "encoding/json" "errors" "fmt" "regexp" "strconv" "strings" + + "github.com/pilosa/pilosa/internal" ) var schemeRegexp = regexp.MustCompile("^[+a-z]+$") @@ -40,9 +43,9 @@ var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0- // localhost // :10101 type URI struct { - scheme string - host string - port uint16 + scheme string `json:"scheme"` + host string `json:"host"` + port uint16 `json:"port"` } // DefaultURI creates and returns the default URI. @@ -54,6 +57,16 @@ func DefaultURI() *URI { } } +type URIs []URI + +func (u URIs) HostPortStrings() []string { + s := make([]string, len(u)) + for i, a := range u { + s[i] = a.HostPort() + } + return s +} + // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { uri := DefaultURI() @@ -67,11 +80,7 @@ func NewURIFromHostPort(host string, port uint16) (*URI, error) { // NewURIFromAddress parses the passed address and returns a URI. func NewURIFromAddress(address string) (*URI, error) { - uri, err := parseAddress(address) - if err != nil { - return nil, err - } - return uri, err + return parseAddress(address) } // Scheme returns the scheme of this URI. @@ -144,9 +153,7 @@ func (u URI) Equals(other *URI) bool { if other == nil { return false } - return u.scheme == other.scheme && - u.host == other.host && - u.port == other.port + return u == *other } // Path returns URI with path @@ -198,3 +205,82 @@ func parseAddress(address string) (uri *URI, err error) { } return uri, nil } + +// Encode converts o into its internal representation. +func (u URI) Encode() *internal.URI { + return encodeURI(u) +} + +func encodeURI(u URI) *internal.URI { + return &internal.URI{ + Scheme: u.scheme, + Host: u.host, + Port: uint32(u.port), + } +} + +func DecodeURI(i *internal.URI) URI { + return decodeURI(i) +} + +func decodeURI(i *internal.URI) URI { + if i == nil { + return URI{} + } + return URI{ + scheme: i.Scheme, + host: i.Host, + port: uint16(i.Port), + } +} + +func encodeURIs(a []URI) []*internal.URI { + if len(a) == 0 { + return nil + } + other := make([]*internal.URI, len(a)) + for i := range a { + other[i] = encodeURI(a[i]) + } + return other +} + +func decodeURIs(a []*internal.URI) []URI { + if len(a) == 0 { + return nil + } + other := make([]URI, len(a)) + for i := range a { + other[i] = decodeURI(a[i]) + } + return other +} + +// MarshalJSON marshals URI into a JSON-encoded byte slice. +func (u *URI) MarshalJSON() ([]byte, error) { + var output struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + output.Scheme = u.scheme + output.Host = u.host + output.Port = u.port + + return json.Marshal(output) +} + +func (u *URI) UnmarshalJSON(b []byte) error { + var input struct { + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Port uint16 `json:"port,omitempty"` + } + if err := json.Unmarshal(b, &input); err != nil { + return err + } + u.scheme = input.Scheme + u.host = input.Host + u.port = input.Port + return nil +} diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 000000000..05afeac19 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,425 @@ +package pilosa + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. +func NewTestCluster(n int) *Cluster { + path, err := ioutil.TempDir("", "pilosa-cluster-") + if err != nil { + panic(err) + } + + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + + for i := 0; i < n; i++ { + c.Nodes = append(c.Nodes, &Node{ + ID: fmt.Sprintf("node%d", i), + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + } + + c.Node = c.Nodes[0] + c.Coordinator = c.Nodes[0].ID + c.SetState(ClusterStateNormal) + + return c +} + +// NewTestURI is a test URI creator that intentionally swallows errors. +func NewTestURI(scheme, host string, port uint16) URI { + uri := DefaultURI() + uri.SetScheme(scheme) + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +func NewTestURIFromHostPort(host string, port uint16) URI { + uri := DefaultURI() + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +// ModHasher represents a simple, mod-based hashing. +type TestModHasher struct{} + +// NewTestModHasher returns a new instance of ModHasher with n buckets. +func NewTestModHasher() *TestModHasher { return &TestModHasher{} } + +func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } + +// ClusterCluster represents a cluster of test nodes, each of which +// has a Cluster. +// ClusterCluster implements Broadcaster interface. +type ClusterCluster struct { + Clusters []*Cluster + + common *commonClusterSettings + + mu sync.RWMutex + resizing bool + resizeDone chan struct{} +} + +type commonClusterSettings struct { + Nodes []*Node +} + +func (t *ClusterCluster) CreateIndex(name string) error { + for _, c := range t.Clusters { + if _, err := c.Holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil { + return err + } + } + return nil +} + +func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { + for _, c := range t.Clusters { + idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) + if err != nil { + return err + } + if _, err := idx.CreateFrame(frame, opt); err != nil { + return err + } + } + return nil +} + +func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { + // Determine which node should receive the SetBit. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := colID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetBit(view, rowID, colID, x) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { + // Determine which node should receive the SetFieldValue. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := columnID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetFieldValue(columnID, name, value) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) clusterByID(id string) *Cluster { + for _, c := range t.Clusters { + if c.Node.ID == id { + return c + } + } + return nil +} + +// AddNode adds a node to the cluster and (potentially) starts a resize job. +func (t *ClusterCluster) AddNode(saveTopology bool) error { + id := len(t.Clusters) + + c, err := t.addCluster(id, saveTopology) + if err != nil { + return err + } + + // Send NodeJoin event to coordinator. + if id > 0 { + coord := t.Clusters[0] + ev := &NodeEvent{ + Event: NodeJoin, + Node: c.Node, + } + + if err := coord.ReceiveEvent(ev); err != nil { + return err + } + + // Wait for the AddNode job to finish. + if c.State() != ClusterStateNormal { + t.resizeDone = make(chan struct{}) + t.mu.Lock() + t.resizing = true + t.mu.Unlock() + <-t.resizeDone + } + } + + return nil +} + +// WriteTopology writes the given topology to disk. +func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { + if buf, err := proto.Marshal(top.Encode()); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) { + + id := fmt.Sprintf("node%d", i) + uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)) + + node := &Node{ + ID: id, + URI: uri, + } + + // add URI to common + //t.common.NodeIDs = append(t.common.NodeIDs, id) + //sort.Sort(t.common.NodeIDs) + + // add node to common + t.common.Nodes = append(t.common.Nodes, node) + + // create node-specific temp directory + path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + if err != nil { + return nil, err + } + + // holder + h := NewHolder() + h.Path = path + + // cluster + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + c.Holder = h + c.MemberSet = NewStaticMemberSet(c.Nodes) + c.Node = node + c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator + c.Broadcaster = t + + // add nodes + if saveTopology { + for _, n := range t.common.Nodes { + c.AddNode(n) + } + } + + // Add this node to the ClusterCluster. + t.Clusters = append(t.Clusters, c) + + return c, nil +} + +// NewClusterCluster returns a new instance of test.Cluster. +func NewClusterCluster(n int) *ClusterCluster { + + tc := &ClusterCluster{ + common: &commonClusterSettings{}, + } + + // add clusters + for i := 0; i < n; i++ { + _, err := tc.addCluster(i, true) + if err != nil { + panic(err) + } + } + return tc +} + +// SetState sets the state of the cluster on each node. +func (t *ClusterCluster) SetState(state string) { + for _, c := range t.Clusters { + c.SetState(state) + } +} + +// Open opens all clusters in the test cluster. +func (t *ClusterCluster) Open() error { + for _, c := range t.Clusters { + if err := c.Open(); err != nil { + return err + } + if err := c.Holder.Open(); err != nil { + return err + } + if err := c.SetNodeState(NodeStateReady); err != nil { + return err + } + } + + // Start the listener on the coordinator. + if len(t.Clusters) == 0 { + return nil + } + t.Clusters[0].ListenForJoins() + + return nil +} + +// Close closes all clusters in the test cluster. +func (t *ClusterCluster) Close() error { + for _, c := range t.Clusters { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + +// SendSync is a test implemenetation of Broadcaster SendSync method. +func (t *ClusterCluster) SendSync(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ClusterStatus: + // Apply the send message to all nodes (except the coordinator). + for _, c := range t.Clusters { + c.MergeClusterStatus(obj) + } + t.mu.RLock() + if obj.State == ClusterStateNormal && t.resizing { + close(t.resizeDone) + } + t.mu.RUnlock() + } + + return nil +} + +// SendAsync is a test implemenetation of Broadcaster SendAsync method. +func (t *ClusterCluster) SendAsync(pb proto.Message) error { + return nil +} + +// SendTo is a test implemenetation of Broadcaster SendTo method. +func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ResizeInstruction: + err := t.FollowResizeInstruction(obj) + if err != nil { + return err + } + case *internal.ResizeInstructionComplete: + coord := t.clusterByID(to.ID) + go coord.MarkResizeInstructionComplete(obj) + } + return nil +} + +// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Node: instr.Node, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrNode := DecodeNode(instr.Node) + destCluster := t.clusterByID(instrNode.ID) + + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } + + for _, src := range instr.Sources { + srcNode := DecodeNode(src.Node) + srcCluster := t.clusterByID(srcNode.ID) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { + return err + } + } + + return nil + }(); err != nil { + complete.Error = err.Error() + } + + node := DecodeNode(instr.Coordinator) + if err := t.SendTo(node, complete); err != nil { + return err + } + + return nil +} diff --git a/view.go b/view.go index 5da97d644..dbcc73b8d 100644 --- a/view.go +++ b/view.go @@ -16,8 +16,6 @@ package pilosa import ( "fmt" - "io" - "io/ioutil" "os" "path/filepath" "strconv" @@ -62,8 +60,8 @@ type View struct { broadcaster Broadcaster stats StatsClient - RowAttrStore *AttrStore - LogOutput io.Writer + RowAttrStore AttrStore + Logger Logger } // NewView returns a new instance of View. @@ -80,7 +78,7 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View { broadcaster: NopBroadcaster, stats: NopStatsClient, - LogOutput: ioutil.Discard, + Logger: NopLogger, } } @@ -98,6 +96,12 @@ func (v *View) Path() string { return v.path } // Open opens and initializes the view. func (v *View) Open() error { + + // Never keep a cache for field views. + if strings.HasPrefix(v.name, ViewFieldPrefix) { + v.cacheType = CacheTypeNone + } + if err := func() error { // Ensure the view's path exists. if err := os.MkdirAll(v.path, 0777); err != nil { @@ -265,11 +269,41 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { frag := NewFragment(path, v.index, v.frame, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.LogOutput = v.LogOutput + frag.Logger = v.Logger frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag } +// DeleteFragment removes the fragment from the view. +func (v *View) DeleteFragment(slice uint64) error { + + fragment := v.fragments[slice] + if fragment == nil { + return ErrFragmentNotFound + } + + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + + // Close data files before deletion. + if err := fragment.Close(); err != nil { + return err + } + + // Delete fragment file. + if err := os.Remove(fragment.Path()); err != nil { + return err + } + + // Delete fragment cache file. + if err := os.Remove(fragment.CachePath()); err != nil { + v.Logger.Printf("no cache file to delete for slice %d", slice) + } + + delete(v.fragments, slice) + + return nil +} + // SetBit sets a bit within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth @@ -323,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e return sum, count, nil } +// FieldMin returns the min and count of a field. +func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + var minHasValue bool + for _, f := range v.Fragments() { + fmin, fcount, err := f.FieldMin(filter, bitDepth) + if err != nil { + return min, count, err + } + // Don't consider a min based on zero columns. + if fcount == 0 { + continue + } + + if !minHasValue { + min = fmin + minHasValue = true + count += fcount + continue + } + + if fmin < min { + min = fmin + count += fcount + } + } + return min, count, nil +} + +// FieldMax returns the max and count of a field. +func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.Fragments() { + fmax, fcount, err := f.FieldMax(filter, bitDepth) + if err != nil { + return max, count, err + } + if fcount > 0 && fmax > max { + max = fmax + count += fcount + } + } + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap() diff --git a/view_test.go b/view_test.go index 5f5ce94e2..87ed8e628 100644 --- a/view_test.go +++ b/view_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "io/ioutil" "os" + "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" @@ -25,22 +26,21 @@ import ( // View is a test wrapper for pilosa.View. type View struct { *pilosa.View - RowAttrStore *test.AttrStore + RowAttrStore pilosa.AttrStore } // NewView returns a new instance of View with a temporary path. func NewView(index, frame, name string) *View { - file, err := ioutil.TempFile("", "pilosa-view-") + path, err := ioutil.TempDir("", "pilosa-view-") if err != nil { panic(err) } - file.Close() v := &View{ - View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize), + View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), RowAttrStore: test.MustOpenAttrStore(), } - v.View.RowAttrStore = v.RowAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore return v } @@ -68,7 +68,7 @@ func (v *View) Reopen() error { } v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore if err := v.Open(); err != nil { return err } @@ -93,3 +93,36 @@ func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { } } } + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := MustOpenView("i", "f", "v") + defer v.Close() + + slice := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.DeleteFragment(slice) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(slice) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same slice, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} diff --git a/webui/assets/main.js b/webui/assets/main.js index 6a4415a5c..b5ff2ab3c 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -254,12 +254,12 @@ function populate_version() { xhr.onload = function() { var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /(v\d+\.\d+)/.exec(version)[0] + var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] var doc_link = document.getElementById('nav-documentation') doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/') + window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') } - node.innerHTML = version + node.innerHTML = "Pilosa v" + version } xhr.send(null) } @@ -295,12 +295,19 @@ function set_active_pane_by_name(name) { function update_cluster_status() { var xhr = new XMLHttpRequest(); xhr.open('GET', '/status') - status_node = document.getElementById('status') xhr.onload = function() { var status = JSON.parse(xhr.responseText) render_status(status) } xhr.send(null) + + var xhrSchema = new XMLHttpRequest(); + xhrSchema.open('GET', '/schema') + xhrSchema.onload = function() { + var schema = JSON.parse(xhrSchema.responseText) + render_schema(schema) + } + xhrSchema.send(null) } function render_status(status) { @@ -310,7 +317,7 @@ function render_status(status) { nodes_div.removeChild(nodes_div.firstChild); } - var nodes = status["status"]["Nodes"] + var nodes = status["nodes"] table = document.createElement("table") tbody = document.createElement("tbody") table.appendChild(tbody) @@ -320,31 +327,35 @@ function render_status(status) { var header = document.createElement('tr') markup = `Host - State` + ID + Coordinator` header.innerHTML = markup tbody.appendChild(header) for(var n=0; n${nodes[n]["Host"]} - ${nodes[n]["State"]}` + markup = `${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} + ${nodes[n]["id"]} + ${nodes[n]["isCoordinator"]}` row.innerHTML = markup tbody.appendChild(row) } nodes_div.appendChild(table) +} +function render_schema(schema) { // render index tables var indexes_div = document.getElementById("status-indexes") while (indexes_div.firstChild) { indexes_div.removeChild(indexes_div.firstChild); } - var indexes = nodes[0]["Indexes"] // TODO currently comes from only node 0 + var indexes = schema["indexes"] // TODO currently comes from only node 0 for(var n=0; n${frames[m]["Name"]} - ${frames[m]["Meta"]["CacheType"]} - ${frames[m]["Meta"]["CacheSize"]}` + row.innerHTML = `${frames[m]["name"]} + ${frames[m]["options"]["cacheType"]} + ${frames[m]["options"]["cacheSize"]}` tbody.appendChild(row) } } @@ -472,7 +483,7 @@ class Autocompleter { } init_dynamic_keywords() { - // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list + // hit /schema, parse indexes, frames, add to list } add_keyword() { @@ -583,4 +594,4 @@ function parse_options(option_str) { } } return options; -} \ No newline at end of file +} diff --git a/webui/index.html b/webui/index.html index dc06e310a..75eaa1d5a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -55,9 +55,9 @@
Special commands
- :create index test [columnLabel=column]
+ :create index test
:use test
- :create frame foo [rowLabel=row]
+ :create frame foo
:delete index test
:delete frame foo