diff --git a/.circleci/config.yml b/.circleci/config.yml index 605990e5b..2ba2868e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 defaults: &defaults working_directory: /go/src/github.com/pilosa/pilosa docker: - - image: circleci/golang:1.10 + - image: circleci/golang:1.11 fast-checkout: &fast-checkout attach_workspace: at: . @@ -30,20 +30,28 @@ jobs: - run: gometalinter --install - run: go get github.com/remyoudompheng/go-misc/deadcode - run: make gometalinter - test-golang-1.10: &base-test + test-golang-1.11: &base-test <<: *defaults steps: - *fast-checkout - run: sudo apt-get install lsof - run: make test - test-golang-1.11-rc: - <<: *base-test - docker: - - image: circleci/golang:1.11-rc - test-golang-1.10-386: + test-golang-1.11-race: + <<: *defaults + steps: + - *fast-checkout + - run: sudo apt-get install lsof + - run: + command: make test TESTFLAGS="-race -timeout=30m" + no_output_timeout: 30m + test-golang-1.11-386: <<: *base-test environment: GOARCH: 386 + test-golang-1.10: + <<: *base-test + docker: + - image: circleci/golang:1.10 cluster-tests: <<: *defaults steps: @@ -96,26 +104,29 @@ workflows: - linter: requires: - build + - test-golang-1.11: + requires: + - build + - test-golang-1.11-race: + requires: + - build + - test-golang-1.11-386: + requires: + - build - test-golang-1.10: requires: - build - - test-golang-1.11-rc: - requires: - - build - - test-golang-1.10-386: - requires: - - build - cluster-tests: requires: - build - prerelease: requires: - linter - - test-golang-1.10 + - test-golang-1.11 - release: requires: - linter - - test-golang-1.10 + - test-golang-1.11 filters: tags: only: /^v.*/ @@ -127,4 +138,4 @@ workflows: - dockerhub-upload: requires: - linter - - test-golang-1.10 + - test-golang-1.11 diff --git a/Gopkg.lock b/Gopkg.lock index 280f6d056..0850ffbaa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,141 +3,192 @@ [[projects]] branch = "master" + digest = "1:7deffcad61694d0cf57178c7d0c41df6b1357612e215914e35cf8b6353abf65c" name = "github.com/CAFxX/gcnotifier" packages = ["."] + pruneopts = "" revision = "39b0596a2da3c92787b3319c6b5425a474b4e0da" [[projects]] branch = "master" + digest = "1:7519fc1e7fa9cde38634cfaf8a2ba842ea20aab2c14d81d6122ab8d6b1658d1c" name = "github.com/DataDog/datadog-go" packages = ["statsd"] + pruneopts = "" revision = "ef3a9daf849df2d7ee3bbf13808dfb481069a773" [[projects]] + digest = "1:f82b8ac36058904227087141017bb82f4b0fc58272990a4cdae3e2d6d222644e" name = "github.com/StackExchange/wmi" packages = ["."] + pruneopts = "" revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338" version = "1.0.0" [[projects]] branch = "master" + digest = "1:354e62d5acb9af138e13ec842f78a846d214a8d4a9f80e578698f1f1565e2ef8" name = "github.com/armon/go-metrics" packages = ["."] + pruneopts = "" revision = "3c58d8115a78a6879e5df75ae900846768d36895" [[projects]] + digest = "1:ed112122ed4a920d944cc99b9d00b0441c11685939c28462c719488d36fe29aa" name = "github.com/boltdb/bolt" packages = ["."] + pruneopts = "" revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" [[projects]] + digest = "1:1660bb2e30cca08494f29b5593e387c6090fbe8936970ba947185b0ca000aec0" name = "github.com/cespare/xxhash" packages = ["."] + pruneopts = "" revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" version = "v1.0.0" [[projects]] + branch = "master" + digest = "1:c46fd324e7902268373e1b337436a6377c196e2dbd7b35624c6256d29d494e78" + name = "github.com/codahale/hdrhistogram" + packages = ["."] + pruneopts = "" + revision = "3a0bb77429bd3a61596f5e8a3172445844342120" + +[[projects]] + digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd" name = "github.com/fsnotify/fsnotify" packages = ["."] + pruneopts = "" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" [[projects]] + digest = "1:96c4a6ff4206086347bfe28e96e092642882128f45ecb8dc8f15f3e6f6703af0" name = "github.com/go-ole/go-ole" packages = [ ".", - "oleutil" + "oleutil", ] + pruneopts = "" revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506" version = "v1.2.1" [[projects]] + digest = "1:6e73003ecd35f4487a5e88270d3ca0a81bc80dc88053ac7e4dcfec5fba30d918" name = "github.com/gogo/protobuf" packages = ["proto"] + pruneopts = "" revision = "636bf0302bc95575d69441b25a2603156ffdddf1" version = "v1.1.1" [[projects]] + digest = "1:f958a1c137db276e52f0b50efee41a1a389dcdded59a69711f3e872757dab34b" name = "github.com/golang/protobuf" packages = ["proto"] + pruneopts = "" revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" version = "v1.1.0" [[projects]] + digest = "1:f9f45f75f332e03fc7e9fe9188ea4e1ce4d14779ef34fa1b023da67518e36327" name = "github.com/google/go-cmp" packages = [ "cmp", "cmp/cmpopts", "cmp/internal/diff", "cmp/internal/function", - "cmp/internal/value" + "cmp/internal/value", ] + pruneopts = "" revision = "3af367b6b30c263d47e8895973edcca9a49cf029" version = "v0.2.0" [[projects]] + digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28" name = "github.com/gorilla/context" packages = ["."] + pruneopts = "" revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" version = "v1.1.1" [[projects]] + digest = "1:a1a1522a8c1fad5675ce8ec9de96f30b898df7c8c229928c35dbd8cf277e6d62" name = "github.com/gorilla/handlers" packages = ["."] + pruneopts = "" revision = "90663712d74cb411cbef281bc1e08c19d1a76145" version = "v1.3.0" [[projects]] + digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1" name = "github.com/gorilla/mux" packages = ["."] + pruneopts = "" revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" version = "v1.6.2" [[projects]] branch = "master" + digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8" name = "github.com/hashicorp/errwrap" packages = ["."] + pruneopts = "" revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc" [[projects]] branch = "master" + digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2" name = "github.com/hashicorp/go-immutable-radix" packages = ["."] + pruneopts = "" revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf" [[projects]] branch = "master" + digest = "1:6396690228a7560bf9247cb90e5ae9c797bd630b01e7d2acab430bbca9a1ecb3" name = "github.com/hashicorp/go-msgpack" packages = ["codec"] + pruneopts = "" revision = "fa3f63826f7c23912c15263591e65d54d080b458" [[projects]] branch = "master" + digest = "1:0b5ca7d18e4ded1e4dacbb37ff027cb40a80c0fed969e4e03cf7aff129bc1b44" name = "github.com/hashicorp/go-multierror" packages = ["."] + pruneopts = "" revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703" [[projects]] branch = "master" + digest = "1:fd8ec2359315965bb6b84fd8e45cd5e8b58b80d8430dc96c8c5dfce46d30dbfc" name = "github.com/hashicorp/go-sockaddr" packages = ["."] + pruneopts = "" revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9" [[projects]] branch = "master" + digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94" name = "github.com/hashicorp/golang-lru" packages = ["simplelru"] + pruneopts = "" revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] branch = "master" + digest = "1:9b7c5846d70f425d7fe279595e32a20994c6075e87be03b5c367ed07280877c5" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -149,142 +200,207 @@ "hcl/token", "json/parser", "json/scanner", - "json/token" + "json/token", ] + pruneopts = "" revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] + digest = "1:d2c45a353b65012162c7ca22c39b1b0bd06d39362fb375cf42b4e48e1104bfc6" name = "github.com/hashicorp/memberlist" packages = ["."] + pruneopts = "" revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28" version = "v0.1.0" [[projects]] + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" name = "github.com/inconshreveable/mousetrap" packages = ["."] + pruneopts = "" revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" [[projects]] + digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5" name = "github.com/magiconair/properties" packages = ["."] + pruneopts = "" revision = "c2353362d570a7bfa228149c62842019201cfb71" version = "v1.8.0" [[projects]] + digest = "1:4c8d8358c45ba11ab7bb15df749d4df8664ff1582daead28bae58cf8cbe49890" name = "github.com/miekg/dns" packages = ["."] + pruneopts = "" revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" version = "v1.0.8" [[projects]] branch = "master" + digest = "1:f43ed2c836208c14f45158fd01577c985688a4d11cf9fd475a939819fef3b321" name = "github.com/mitchellh/mapstructure" packages = ["."] + pruneopts = "" revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac" [[projects]] + digest = "1:78fb99d6011c2ae6c72f3293a83951311147b12b06a5ffa43abf750c4fab6ac5" + name = "github.com/opentracing/opentracing-go" + packages = [ + ".", + "ext", + "log", + ] + pruneopts = "" + revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" + version = "v1.0.2" + +[[projects]] + digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457" name = "github.com/pelletier/go-toml" packages = ["."] + pruneopts = "" revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" version = "v1.2.0" [[projects]] - name = "github.com/pilosa/go-pilosa" - packages = [ - ".", - "gopilosa_pbuf" - ] - revision = "4e7807f5ad779407936744057cd17332046b6c3c" - version = "v1.1.0" - -[[projects]] + digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:7f569d906bdd20d906b606415b7d794f798f91a62fcfb6a4daa6d50690fb7a3f" name = "github.com/satori/go.uuid" packages = ["."] + pruneopts = "" revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" version = "v1.2.0" [[projects]] branch = "master" + digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7" name = "github.com/sean-/seed" packages = ["."] + pruneopts = "" revision = "e2103e2c35297fb7e17febb81e49b312087a2372" [[projects]] + digest = "1:02715a2fb4b9279af36651a59a51dd4164eb689bd6785874811899f43eeb2a54" name = "github.com/shirou/gopsutil" packages = [ - "cpu", "host", "internal/common", "mem", - "net", - "process" + "process", ] + pruneopts = "" revision = "8048a2e9c5773235122027dd585cf821b2af1249" version = "v2.18.07" [[projects]] - branch = "master" - name = "github.com/shirou/w32" - packages = ["."] - revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" - -[[projects]] + digest = "1:7ba2551c9a8de293bc575dbe2c0d862c52252d26f267f784547f059f512471c8" name = "github.com/spf13/afero" packages = [ ".", - "mem" + "mem", ] + pruneopts = "" revision = "787d034dfe70e44075ccc060d346146ef53270ad" version = "v1.1.1" [[projects]] + digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4" name = "github.com/spf13/cast" packages = ["."] + pruneopts = "" revision = "8965335b8c7107321228e3e3702cab9832751bac" version = "v1.2.0" [[projects]] + digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6" name = "github.com/spf13/cobra" packages = ["."] + pruneopts = "" revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" version = "v0.0.3" [[projects]] branch = "master" + digest = "1:104517520aab91164020ab6524a5d6b7cafc641b2e42ac6236f6ac1deac4f66a" name = "github.com/spf13/jwalterweatherman" packages = ["."] + pruneopts = "" revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" [[projects]] + digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [[projects]] + digest = "1:3dab237cd3263a290d771d133fed777bb56c22e380b00ebe92e6531d5c8d3d0c" name = "github.com/spf13/viper" packages = ["."] + pruneopts = "" revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" version = "v1.0.2" +[[projects]] + digest = "1:941ab4973b3218a9a6d02d31734f5c762239eb538c0d702fff62d4037af8ab0a" + name = "github.com/uber/jaeger-client-go" + packages = [ + ".", + "config", + "internal/baggage", + "internal/baggage/remote", + "internal/spanlog", + "internal/throttler", + "internal/throttler/remote", + "log", + "rpcmetrics", + "thrift", + "thrift-gen/agent", + "thrift-gen/baggage", + "thrift-gen/jaeger", + "thrift-gen/sampling", + "thrift-gen/zipkincore", + "transport", + "utils", + ] + pruneopts = "" + revision = "1a782e2da844727691fef1757c72eb190c2909f0" + version = "v2.15.0" + +[[projects]] + digest = "1:aa1598d34009b45ce74fdabdd25e4258d7923d1e1b418d4c98482e79607cb9b0" + name = "github.com/uber/jaeger-lib" + packages = ["metrics"] + pruneopts = "" + revision = "ed3a127ec5fef7ae9ea95b01b542c47fbd999ce5" + version = "v1.5.0" + [[projects]] branch = "master" + digest = "1:cae234a803b78380e4d769db6036b9fcc8c08ed4ff862571ffc1a958edc1f629" name = "golang.org/x/crypto" packages = [ "ed25519", - "ed25519/internal/edwards25519" + "ed25519/internal/edwards25519", ] + pruneopts = "" revision = "c126467f60eb25f8f27e5a981f32a87e3965053f" [[projects]] branch = "master" + digest = "1:1f3b488bf9c50cac0cb738fca05d8ff55509c8698f58ea4d74b1499d0baaeb74" name = "golang.org/x/net" packages = [ "bpf", @@ -292,26 +408,32 @@ "internal/iana", "internal/socket", "ipv4", - "ipv6" + "ipv6", ] + pruneopts = "" revision = "22bb95c5e783d192c577a7b310b06637db9f1d94" [[projects]] branch = "master" + digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1" name = "golang.org/x/sync" packages = ["errgroup"] + pruneopts = "" revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" [[projects]] branch = "master" + digest = "1:dd631ee90bd2e7aa16b6e094217d77a797684b52811374c948c695cbb46b5bbb" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", ] + pruneopts = "" revision = "bd9dbc187b6e1dacfdd2722a87e83093c2d7bd6e" [[projects]] + digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" name = "golang.org/x/text" packages = [ "internal/gen", @@ -319,20 +441,49 @@ "internal/ucd", "transform", "unicode/cldr", - "unicode/norm" + "unicode/norm", ] + pruneopts = "" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] + digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "" revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" version = "v2.2.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "be318fa4f2a72e7e849b2faff1ce9300deaaf2d24cf76100f4b538abed86295b" + input-imports = [ + "github.com/CAFxX/gcnotifier", + "github.com/DataDog/datadog-go/statsd", + "github.com/boltdb/bolt", + "github.com/cespare/xxhash", + "github.com/davecgh/go-spew/spew", + "github.com/gogo/protobuf/proto", + "github.com/golang/protobuf/proto", + "github.com/google/go-cmp/cmp", + "github.com/google/go-cmp/cmp/cmpopts", + "github.com/gorilla/handlers", + "github.com/gorilla/mux", + "github.com/hashicorp/memberlist", + "github.com/opentracing/opentracing-go", + "github.com/opentracing/opentracing-go/ext", + "github.com/pelletier/go-toml", + "github.com/pkg/errors", + "github.com/satori/go.uuid", + "github.com/shirou/gopsutil/host", + "github.com/shirou/gopsutil/mem", + "github.com/spf13/cobra", + "github.com/spf13/pflag", + "github.com/spf13/viper", + "github.com/uber/jaeger-client-go", + "github.com/uber/jaeger-client-go/config", + "golang.org/x/sync/errgroup", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 34f97965e..af86b8241 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-dep install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build require-dep require-gometalinter require-protoc require-protoc-gen-gofast require-peg test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-dep install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -150,26 +150,10 @@ gometalinter: require-gometalinter ###################### # Verifies that needed build dependency is installed. Errors out if not installed. -define require - $(if $(shell command -v $1 2>/dev/null), - $(info Verified build dependency "$1" is installed.), - $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`)) -endef - -require-dep: - $(call require,dep) - -require-protoc-gen-gofast: - $(call require,protoc-gen-gofast) - -require-protoc: - $(call require,protoc) - -require-peg: - $(call require,peg) - -require-gometalinter: - $(call require,gometalinter) +require-%: + $(if $(shell command -v $* 2>/dev/null),\ + $(info Verified build dependency "$*" is installed.),\ + $(error Build dependency "$*" not installed. To install, try `make install-$*`)) install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg diff --git a/api.go b/api.go index 7a95ce76e..92ac7672e 100644 --- a/api.go +++ b/api.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -99,6 +100,9 @@ func (api *API) validate(f apiMethod) error { // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "API.Query") + defer span.Finish() + if err := api.validate(apiQuery); err != nil { return QueryResponse{}, errors.Wrap(err, "validating api method") } @@ -122,7 +126,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } // CreateIndex makes a new Pilosa index. -func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) { +func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.CreateIndex") + defer span.Finish() + if err := api.validate(apiCreateIndex); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -146,7 +153,10 @@ func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOp } // Index retrieves the named index. -func (api *API) Index(_ context.Context, indexName string) (*Index, error) { +func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.Index") + defer span.Finish() + if err := api.validate(apiIndex); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -160,7 +170,10 @@ func (api *API) Index(_ context.Context, indexName string) (*Index, error) { // DeleteIndex removes the named index. If the index is not found it does // nothing and returns no error. -func (api *API) DeleteIndex(_ context.Context, indexName string) error { +func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteIndex") + defer span.Finish() + if err := api.validate(apiDeleteIndex); err != nil { return errors.Wrap(err, "validating api method") } @@ -186,7 +199,10 @@ func (api *API) DeleteIndex(_ context.Context, indexName string) error { // CreateField makes the named field in the named index with the given options. // This method currently only takes a single functional option, but that may be // changed in the future to support multiple options. -func (api *API) CreateField(_ context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { +func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField") + defer span.Finish() + if err := api.validate(apiCreateField); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -228,7 +244,10 @@ func (api *API) CreateField(_ context.Context, indexName string, fieldName strin } // Field retrieves the named field. -func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, error) { +func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.Field") + defer span.Finish() + if err := api.validate(apiField); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -268,21 +287,14 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { // (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all // of the rows in this shard of this field concatenated together in one long // bitmap. -func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, data []byte, opts ...ImportOption) (err error) { - if len(data) == 0 { - return errors.New("no data to import") - } +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) { + span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring") + defer span.Finish() if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } - // Set up import options. - options, err := setUpImportOptions(opts...) - if err != nil { - return errors.Wrap(err, "setting up import options") - } - nodes := api.cluster.shardNodes(indexName, shard) var eg errgroup.Group @@ -291,26 +303,42 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return newNotFoundError(ErrFieldNotFound) } - // only set fields are supported - if field.Type() != FieldTypeSet { - return NewBadRequestError(errors.New("roaring import is only supported for set fields")) + // only set and time fields are supported + if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime { + return NewBadRequestError(errors.New("roaring import is only supported for set and time fields")) } for _, node := range nodes { node := node if node.ID == api.server.nodeID { - // must make a copy of data to operate on locally. field.importRoaring changes data - d2 := make([]byte, len(data)) - copy(d2, data) eg.Go(func() error { - return field.importRoaring(d2, shard, options.Clear) + var err error + for viewName, viewData := range req.Views { + if viewName == "" { + viewName = viewStandard + } else { + viewName = fmt.Sprintf("%s_%s", viewStandard, viewName) + } + if len(viewData) == 0 { + return fmt.Errorf("no data to import for view: %s", viewName) + } + // must make a copy of data to operate on locally. + // field.importRoaring changes data + data := make([]byte, len(viewData)) + copy(data, viewData) + err = field.importRoaring(data, shard, viewName, req.Clear) + if err != nil { + return err + } + } + return err }) go func(node *Node) { }(node) } else if !remote { // if remote == true we don't forward to other nodes // forward it on eg.Go(func() error { - return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data, opts...) + return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, req) }) } } @@ -320,7 +348,10 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, // DeleteField removes the named field from the named index. If the index is not // found, an error is returned. If the field is not found, it is ignored and no // action is taken. -func (api *API) DeleteField(_ context.Context, indexName string, fieldName string) error { +func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteField") + defer span.Finish() + if err := api.validate(apiDeleteField); err != nil { return errors.Wrap(err, "validating api method") } @@ -384,7 +415,10 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str // ExportCSV encodes the fragment designated by the index,field,shard as // CSV of the form , -func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.ExportCSV") + defer span.Finish() + if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } @@ -418,6 +452,7 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, // Define the function to write each bit as a string, // translating to keys where necessary. + var n int fn := func(rowID, columnID uint64) error { var rowStr string var colStr string @@ -439,6 +474,7 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, colStr = strconv.FormatUint(columnID, 10) } + n++ return cw.Write([]string{rowStr, colStr}) } @@ -450,11 +486,16 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, // Ensure data is flushed. cw.Flush() + span.LogKV("n", n) + return nil } // ShardNodes returns the node and all replicas which should contain a shard's data. -func (api *API) ShardNodes(_ context.Context, indexName string, shard uint64) ([]*Node, error) { +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes") + defer span.Finish() + if err := api.validate(apiShardNodes); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -465,7 +506,10 @@ func (api *API) ShardNodes(_ context.Context, indexName string, shard uint64) ([ // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. -func (api *API) FragmentBlockData(_ context.Context, body io.Reader) ([]byte, error) { +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlockData") + defer span.Finish() + if err := api.validate(apiFragmentBlockData); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -498,7 +542,10 @@ func (api *API) FragmentBlockData(_ context.Context, body io.Reader) ([]byte, er } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(_ context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlocks") + defer span.Finish() + if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -516,7 +563,9 @@ func (api *API) FragmentBlocks(_ context.Context, indexName, fieldName, viewName // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. -func (api *API) Hosts(_ context.Context) []*Node { +func (api *API) Hosts(ctx context.Context) []*Node { + span, _ := tracing.StartSpanFromContext(ctx, "API.Hosts") + defer span.Finish() return api.cluster.Nodes() } @@ -527,7 +576,10 @@ func (api *API) Node() *Node { } // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. -func (api *API) RecalculateCaches(_ context.Context) error { +func (api *API) RecalculateCaches(ctx context.Context) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.RecalculateCaches") + defer span.Finish() + if err := api.validate(apiRecalculateCaches); err != nil { return errors.Wrap(err, "validating api method") } @@ -542,7 +594,10 @@ func (api *API) RecalculateCaches(_ context.Context) error { // PostClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. -func (api *API) ClusterMessage(_ context.Context, reqBody io.Reader) error { +func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.ClusterMessage") + defer span.Finish() + if err := api.validate(apiClusterMessage); err != nil { return errors.Wrap(err, "validating api method") } @@ -569,12 +624,17 @@ func (api *API) ClusterMessage(_ context.Context, reqBody io.Reader) error { // Schema returns information about each index in Pilosa including which fields // they contain. -func (api *API) Schema(_ context.Context) []*IndexInfo { +func (api *API) Schema(ctx context.Context) []*IndexInfo { + span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") + defer span.Finish() return api.holder.limitedSchema() } // Views returns the views in the given field. -func (api *API) Views(_ context.Context, indexName string, fieldName string) ([]*view, error) { +func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.Views") + defer span.Finish() + if err := api.validate(apiViews); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -591,7 +651,10 @@ func (api *API) Views(_ context.Context, indexName string, fieldName string) ([] } // DeleteView removes the given view. -func (api *API) DeleteView(_ context.Context, indexName string, fieldName string, viewName string) error { +func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteView") + defer span.Finish() + if err := api.validate(apiDeleteView); err != nil { return errors.Wrap(err, "validating api method") } @@ -625,7 +688,10 @@ func (api *API) DeleteView(_ context.Context, indexName string, fieldName string } // IndexAttrDiff -func (api *API) IndexAttrDiff(_ context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff") + defer span.Finish() + if err := api.validate(apiIndexAttrDiff); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -659,7 +725,10 @@ func (api *API) IndexAttrDiff(_ context.Context, indexName string, blocks []Attr return attrs, nil } -func (api *API) FieldAttrDiff(_ context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff") + defer span.Finish() + if err := api.validate(apiFieldAttrDiff); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -718,6 +787,9 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption { // Import bulk imports data into a particular index,field,shard. func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.Import") + defer span.Finish() + if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } @@ -823,6 +895,9 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error { + span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue") + defer span.Finish() + if err := api.validate(apiImportValue); err != nil { return errors.Wrap(err, "validating api method") } @@ -916,7 +991,10 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error { // MaxShards returns the maximum shard number for each index in a map. // TODO (2.0): This method has been deprecated. Instead, use // AvailableShardsByIndex. -func (api *API) MaxShards(_ context.Context) map[string]uint64 { +func (api *API) MaxShards(ctx context.Context) map[string]uint64 { + span, _ := tracing.StartSpanFromContext(ctx, "API.MaxShards") + defer span.Finish() + m := make(map[string]uint64) for k, v := range api.holder.availableShardsByIndex() { m[k] = v.Max() @@ -925,7 +1003,9 @@ func (api *API) MaxShards(_ context.Context) map[string]uint64 { } // AvailableShardsByIndex returns bitmaps of shards with available by index name. -func (api *API) AvailableShardsByIndex(_ context.Context) map[string]*roaring.Bitmap { +func (api *API) AvailableShardsByIndex(ctx context.Context) map[string]*roaring.Bitmap { + span, _ := tracing.StartSpanFromContext(ctx, "API.AvailableShardsByIndex") + defer span.Finish() return api.holder.availableShardsByIndex() } @@ -957,7 +1037,7 @@ func (api *API) validateShardOwnership(indexName string, shard uint64) error { } func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) { - api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard) + api.server.logger.Debugf("importing: %v %v %v", indexName, fieldName, shard) // Find the Index. index := api.holder.Index(indexName) @@ -976,7 +1056,10 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I } // SetCoordinator makes a new Node the cluster coordinator. -func (api *API) SetCoordinator(_ context.Context, id string) (oldNode, newNode *Node, err error) { +func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.SetCoordinator") + defer span.Finish() + if err := api.validate(apiSetCoordinator); err != nil { return nil, nil, errors.Wrap(err, "validating api method") } @@ -1041,6 +1124,9 @@ func (api *API) ResizeAbort() error { // GetTranslateData provides a reader for key translation logs starting at offset. func (api *API) GetTranslateData(ctx context.Context, offset int64) (io.ReadCloser, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateData") + defer span.Finish() + rc, err := api.holder.translateFile.Reader(ctx, offset) if err != nil { return nil, errors.Wrap(err, "read from translate store") diff --git a/attr.go b/attr.go index 629cbf587..b813696e6 100644 --- a/attr.go +++ b/attr.go @@ -203,25 +203,3 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) { } return decodeAttrs(pb.GetAttrs()), nil } - -// memAttrStore represents an in-memory implementation of the AttrStore interface. -type memAttrStore struct { - store map[uint64]map[string]interface{} -} - -func (s *memAttrStore) Path() string { return "" } -func (s *memAttrStore) Open() error { return nil } -func (s *memAttrStore) Close() error { return nil } -func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } -func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { - s.store[id] = m - return nil -} -func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { - for id, v := range m { - s.store[id] = v - } - return nil -} -func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } -func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } diff --git a/client.go b/client.go index f6c49d14b..b16222b7a 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error + ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error { return nil } -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/cluster.go b/cluster.go index e22781a25..451ce8ba1 100644 --- a/cluster.go +++ b/cluster.go @@ -27,14 +27,14 @@ import ( "sync" "time" - "golang.org/x/sync/errgroup" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/logger" "github.com/pilosa/pilosa/roaring" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" + "golang.org/x/sync/errgroup" ) const ( @@ -347,8 +347,6 @@ func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool { // addNode adds a node to the Cluster and updates and saves the // new topology. unprotected. func (c *cluster) addNode(node *Node) error { - c.logger.Printf("add node %s to cluster on %s", node, c.Node) - // If the node being added is the coordinator, set it for this node. if node.IsCoordinator { c.Coordinator = node.ID @@ -481,7 +479,7 @@ func (c *cluster) setNodeState(state string) error { // nolint: unparam State: state, } - c.logger.Printf("Sending State %s (%s)", state, c.Coordinator) + c.logger.Printf("sending state %s (%s)", state, c.Coordinator) if err := c.sendTo(c.coordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } @@ -970,7 +968,6 @@ func (c *cluster) close() error { } func (c *cluster) markAsJoined() { - c.logger.Printf("mark node as joined (received coordinator update)") if !c.joined { c.joined = true close(c.joining) @@ -1069,7 +1066,6 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error { } // Broadcast cluster status changes to the cluster. status := c.unprotectedStatus() - c.logger.Printf("broadcasting ClusterStatus: %s", status) return c.broadcaster.SendSync(status) // TODO fix c.Status } @@ -1246,14 +1242,14 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { return errors.Wrap(err, "merging cluster status") } - c.logger.Printf("MergeClusterStatus done, start goroutine") + c.logger.Printf("done MergeClusterStatus, start goroutine") // The actual resizing runs in a goroutine because we don't want to block // the distribution of other ResizeInstructions to the rest of the cluster. go func() { // Make sure the holder has opened. - <-c.holder.opened + c.holder.opened.Recv() // Prepare the return message. complete := &ResizeInstructionComplete{ @@ -1264,9 +1260,11 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // Stop processing on any error. if err := func() error { + span, ctx := tracing.StartSpanFromContext(context.Background(), "Cluster.followResizeInstruction") + defer span.Finish() // Sync the schema received in the resize instruction. - c.logger.Printf("Holder ApplySchema") + c.logger.Debugf("holder applySchema") if err := c.holder.applySchema(instr.Schema); err != nil { return errors.Wrap(err, "applying schema") } @@ -1297,7 +1295,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // Stream shard from remote node. c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI) + rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a shard has been skipped and @@ -1651,17 +1649,17 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { switch e.Event { case NodeJoin: - c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI) + c.logger.Debugf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI) // Ignore the event if this is not the coordinator. if !c.isCoordinator() { return nil } return c.nodeJoin(e.Node) case NodeLeave: - c.logger.Printf("received node leave on %s: %s, uri: %v", c.Node, e.Node, e.Node.URI) c.mu.Lock() defer c.mu.Unlock() if c.unprotectedIsCoordinator() { + c.logger.Printf("received node leave: %v", e.Node) // if removeNodeBasicSorted succeeds, that means that the node was // not already removed by a removeNode request. We treat this as the // host being temporarily unavailable, and expect it to come back @@ -1673,7 +1671,6 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { err = c.unprotectedSetStateAndBroadcast(c.determineClusterState()) } } - c.logger.Printf("finished node leave on %s: %s, uri: %v", c.Node, e.Node, e.Node.URI) case NodeUpdate: c.logger.Printf("received node update event: id: %v, string: %v, uri: %v", e.Node.ID, e.Node.String(), e.Node.URI) // NodeUpdate is intentionally not implemented. @@ -1686,7 +1683,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { func (c *cluster) nodeJoin(node *Node) error { c.mu.Lock() defer c.mu.Unlock() - c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID) + c.logger.Printf("node join event on coordinator, node: %s, id: %s", node.URI, node.ID) if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { @@ -1726,7 +1723,7 @@ func (c *cluster) nodeJoin(node *Node) error { // the cluster. if cnode := c.unprotectedNodeByID(node.ID); cnode != nil { if cnode.URI != node.URI { - c.logger.Printf("Node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI) + c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI) cnode.URI = node.URI } return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) diff --git a/cmd/import.go b/cmd/import.go index 878aad286..81a12a47f 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -19,10 +19,8 @@ import ( "io" "github.com/pilosa/pilosa" - - "github.com/spf13/cobra" - "github.com/pilosa/pilosa/ctl" + "github.com/spf13/cobra" ) var Importer *ctl.ImportCommand @@ -55,6 +53,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index") flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field") + flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex") flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation") flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation") flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked") diff --git a/cmd/server.go b/cmd/server.go index d4834672e..6e0e3368a 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -17,11 +17,13 @@ package cmd import ( "io" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/pilosa/pilosa/ctl" "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/tracing" + "github.com/pilosa/pilosa/tracing/opentracing" + "github.com/pkg/errors" + "github.com/spf13/cobra" + jaegercfg "github.com/uber/jaeger-client-go/config" ) // Server is global so that tests can control and verify it. @@ -39,9 +41,28 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { + // Start & run the server. if err := Server.Start(); err != nil { return errors.Wrap(err, "running server") } + + // Initialize tracing in the command since it is global. + var cfg jaegercfg.Configuration + cfg.ServiceName = "pilosa" + cfg.Sampler = &jaegercfg.SamplerConfig{ + Type: Server.Config.Tracing.SamplerType, + Param: Server.Config.Tracing.SamplerParam, + } + cfg.Reporter = &jaegercfg.ReporterConfig{ + LocalAgentHostPort: Server.Config.Tracing.AgentHostPort, + } + tracer, closer, err := cfg.NewTracer() + if err != nil { + return errors.Wrap(err, "initializing jaeger tracer") + } + defer closer.Close() + tracing.GlobalTracer = opentracing.NewTracer(tracer) + return errors.Wrap(Server.Wait(), "waiting on Server") }, } diff --git a/ctl/import.go b/ctl/import.go index d49b50eb3..ab19064ce 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -99,13 +99,15 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { - // set the correct type for the field - if cmd.FieldOptions.TimeQuantum != "" { - cmd.FieldOptions.Type = "time" - } else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 { - cmd.FieldOptions.Type = "int" - } else { - cmd.FieldOptions.Type = "set" + if cmd.FieldOptions.Type == "" { + // set the correct type for the field + if cmd.FieldOptions.TimeQuantum != "" { + cmd.FieldOptions.Type = "time" + } else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 { + cmd.FieldOptions.Type = "int" + } else { + cmd.FieldOptions.Type = "set" + } } err := cmd.ensureSchema(ctx) if err != nil { diff --git a/ctl/server.go b/ctl/server.go index 7f384ec7b..5ed40dbf4 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -68,4 +68,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") + + // Tracing + flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.") + flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.") + flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.") } diff --git a/docs/configuration.md b/docs/configuration.md index 5750441ef..140258437 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -307,6 +307,42 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h skip-verify = true ``` +#### Tracing Sampler Type + +* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote) +* Flag: `tracing.sampler-type` +* Env: `PILOSA_TRACING_SAMPLER_TYPE` +* Config: + + ```toml + [tracing] + sampler-type = "remote" + ``` + +#### Tracing Sampler Parameter + +* Description: Jaeger sampler parameter (number) +* Flag: `tracing.sampler-param` +* Env: `PILOSA_TRACING_SAMPLER_PARAM` +* Config: + + ```toml + [tracing] + sampler-param = 0.001 + ``` + +#### Tracing Agent Host/Port + +* Description: Jaeger agent host:port +* Flag: `tracing.agent-host-port` +* Env: `PILOSA_TRACING_AGENT_HOST_PORT` +* Config: + + ```toml + [tracing] + agent-host-port = "localhost:6831" + ``` + #### Translation Map Size * Description: Size in bytes of mmap to allocate for key translation diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 9826c7ef9..3c7629a65 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -217,6 +217,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeImportValueRequest(msg, mt) return nil + case *pilosa.ImportRoaringRequest: + msg := &internal.ImportRoaringRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRoaringRequest") + } + decodeImportRoaringRequest(msg, mt) + return nil case *pilosa.ImportResponse: msg := &internal.ImportResponse{} err := proto.Unmarshal(buf, msg) @@ -292,6 +300,8 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeImportRequest(mt) case *pilosa.ImportValueRequest: return encodeImportValueRequest(mt) + case *pilosa.ImportRoaringRequest: + return encodeImportRoaringRequest(mt) case *pilosa.ImportResponse: return encodeImportResponse(mt) case *pilosa.BlockDataRequest: @@ -348,6 +358,22 @@ func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValu } } +func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.ImportRoaringRequest { + views := make([]*internal.ImportRoaringRequestView, len(m.Views)) + i := 0 + for viewName, viewData := range m.Views { + views[i] = &internal.ImportRoaringRequestView{ + Name: viewName, + Data: viewData, + } + i += 1 + } + return &internal.ImportRoaringRequest{ + Clear: m.Clear, + Views: views, + } +} + func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { return &internal.QueryRequest{ Query: m.Query, @@ -916,6 +942,15 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV m.Values = pb.Values } +func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { + views := map[string][]byte{} + for _, view := range pb.Views { + views[view.Name] = view.Data + } + m.Clear = pb.Clear + m.Views = views +} + func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { m.Err = pb.Err } diff --git a/executor.go b/executor.go index 918fe9611..edff9f0e6 100644 --- a/executor.go +++ b/executor.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -81,7 +82,11 @@ func newExecutor(opts ...executorOption) *executor { // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") + defer span.Finish() + resp := QueryResponse{} + // Verify that an index is set. if index == "" { return resp, ErrIndexRequired @@ -105,10 +110,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // Translate query keys to ids, if necessary. // No need to translate a remote call. if !opt.Remote { - for i := range q.Calls { - if err := e.translateCall(index, idx, q.Calls[i]); err != nil { - return resp, err - } + if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil { + return resp, err } } @@ -154,11 +157,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // Translate response objects from ids to keys, if necessary. // No need to translate a remote call. if !opt.Remote { - for i := range results { - results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) - if err != nil { - return resp, err - } + if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil { + return resp, err } } @@ -189,6 +189,9 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr } func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute") + defer span.Finish() + // Don't bother calculating shards for query types that don't require it. needsShards := needsShards(q.Calls) @@ -225,6 +228,9 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // executeCall executes a call. func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") + defer span.Finish() + if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } @@ -292,6 +298,9 @@ func (e *executor) validateCallArgs(c *pql.Call) error { } func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeOptionsCall") + defer span.Finish() + optCopy := &execOptions{} *optCopy = *opt if arg, ok := c.Args["columnAttrs"]; ok { @@ -335,6 +344,9 @@ func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql. // executeSum executes a Sum() call. func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSum") + defer span.Finish() + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Sum(): field required") } @@ -368,6 +380,9 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh // executeMin executes a Min() call. func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin") + defer span.Finish() + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Min(): field required") } @@ -401,6 +416,9 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh // executeMax executes a Max() call. func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax") + defer span.Finish() + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Max(): field required") } @@ -434,6 +452,9 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh // executeBitmapCall executes a call that returns a bitmap. func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") + defer span.Finish() + // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { return e.executeBitmapCallShard(ctx, index, c, shard) @@ -500,6 +521,9 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // executeBitmapCallShard executes a bitmap call for a single shard. func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") + defer span.Finish() + switch c.Name { case "Row": return e.executeBitmapShard(ctx, index, c, shard) @@ -524,6 +548,9 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * // executeSumCountShard calculates the sum and count for bsiGroups on a shard. func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard") + defer span.Finish() + var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) @@ -562,6 +589,9 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq // executeMinShard calculates the min for bsiGroups on a shard. func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinShard") + defer span.Finish() + var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) @@ -600,6 +630,9 @@ func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Cal // executeMaxShard calculates the max for bsiGroups on a shard. func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxShard") + defer span.Finish() + var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) @@ -640,6 +673,9 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN") + defer span.Finish() + idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -679,6 +715,9 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s } func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards") + defer span.Finish() + // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { return e.executeTopNShard(ctx, index, c, shard) @@ -704,6 +743,9 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C // executeTopNShard executes a TopN call for a single shard. func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") + defer span.Finish() + field, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { @@ -766,6 +808,9 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca // executeDifferenceShard executes a difference() call for a local shard. func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDifferenceShard") + defer span.Finish() + var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Difference query is currently not supported") @@ -1095,8 +1140,11 @@ func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call return frag.rows(start, filters...), nil } -func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - // Fetch index. +func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapShard") + defer span.Finish() + + // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound @@ -1128,6 +1176,9 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca // executeIntersectShard executes a intersect() call for a local shard. func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") + defer span.Finish() + var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Intersect query is currently not supported") @@ -1150,6 +1201,9 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p // executeRangeShard executes a range() call for a local shard. func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeRangeShard") + defer span.Finish() + // Handle bsiGroup ranges differently. if c.HasConditionArg() { return e.executeBSIGroupRangeShard(ctx, index, c, shard) @@ -1222,7 +1276,10 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C } // executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeBSIGroupRangeShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { +func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBSIGroupRangeShard") + defer span.Finish() + // Only one conditional should be present. if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") @@ -1353,6 +1410,9 @@ func (e *executor) executeBSIGroupRangeShard(_ context.Context, index string, c // executeUnionShard executes a union() call for a local shard. func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") + defer span.Finish() + other := NewRow() for i, input := range c.Children { row, err := e.executeBitmapCallShard(ctx, index, input, shard) @@ -1372,6 +1432,9 @@ func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.C // executeXorShard executes a xor() call for a local shard. func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeXorShard") + defer span.Finish() + other := NewRow() for i, input := range c.Children { row, err := e.executeBitmapCallShard(ctx, index, input, shard) @@ -1391,6 +1454,9 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal // executeNotShard executes a not() call for a local shard. func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") + defer span.Finish() + if len(c.Children) == 0 { return nil, errors.New("Not() requires an input row") } else if len(c.Children) > 1 { @@ -1439,6 +1505,9 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") + defer span.Finish() + if len(c.Children) == 0 { return 0, errors.New("Count() requires an input bitmap") } else if len(c.Children) > 1 { @@ -1471,6 +1540,9 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, // executeClearBit executes a Clear() call. func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBit") + defer span.Finish() + fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Clear() argument required: field") @@ -1506,6 +1578,9 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // executeClearBitField executes a Clear() call for a field. func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBitField") + defer span.Finish() + shard := colID / ShardWidth ret := false for _, node := range e.Cluster.shardNodes(index, shard) { @@ -1536,6 +1611,9 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq // executeClearRow executes a ClearRow() call. func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearRow") + defer span.Finish() + // Ensure the field type supports ClearRow(). fieldName, err := c.FieldArg() if err != nil { @@ -1572,7 +1650,10 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal } // executeClearRowShard executes a ClearRow() call for a single shard. -func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) { +func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeClearRowShard") + defer span.Finish() + fieldName, err := c.FieldArg() if err != nil { return false, errors.New("ClearRow() argument required: field") @@ -1682,7 +1763,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. if err != nil { return false, errors.Wrap(err, "creating view") } - fragment, err = view.createFragmentIfNotExists(shard) + fragment, err = view.CreateFragmentIfNotExists(shard) if err != nil { return false, errors.Wrapf(err, "creating fragment: %d", shard) } @@ -1698,6 +1779,9 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSet") + defer span.Finish() + // Read colID. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { @@ -1765,6 +1849,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op // executeSetBitField executes a Set() call for a specific field. func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetBitField") + defer span.Finish() + shard := colID / ShardWidth ret := false @@ -1797,6 +1884,9 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. // executeSetValueField executes a Set() call for a specific int field. func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetValueField") + defer span.Finish() + shard := colID / ShardWidth ret := false @@ -1829,6 +1919,9 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq // executeSetRowAttrs executes a SetRowAttrs() call. func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetRowAttrs") + defer span.Finish() + fieldName, ok := c.Args["_field"].(string) if !ok { return errors.New("SetRowAttrs() field required") @@ -1886,6 +1979,9 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *execOptions) ([]interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBulkSetRowAttrs") + defer span.Finish() + // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { @@ -1973,6 +2069,9 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // executeSetColumnAttrs executes a SetColumnAttrs() call. func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetColumnAttrs") + defer span.Finish() + // Retrieve index. idx := e.Holder.Index(index) if idx == nil { @@ -2021,6 +2120,9 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // remoteExec executes a PQL query remotely for a set of shards on a node. func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec") + defer span.Finish() + // Encode request object. pbreq := &QueryRequest{ Query: q.String(), @@ -2059,6 +2161,9 @@ loop: // If a mapping of shards to a node fails then the shards are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapReduce") + defer span.Finish() + ch := make(chan mapResponse) // Wrap context with a cancel to kill goroutines on exit. @@ -2073,7 +2178,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.nodes).Clone() } else { - nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)} + nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} } // Start mapping across all primary owners. @@ -2118,6 +2223,9 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") + defer span.Finish() + // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { @@ -2153,6 +2261,9 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod // mapperLocal performs map & reduce entirely on the local node. func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal") + defer span.Finish() + ch := make(chan mapResponse, len(shards)) for _, shard := range shards { @@ -2189,6 +2300,18 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } } +func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, calls []*pql.Call) error { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateCalls") + defer span.Finish() + + for i := range calls { + if err := e.translateCall(index, idx, calls[i]); err != nil { + return err + } + } + return nil +} + func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { var colKey, rowKey, fieldName string switch c.Name { @@ -2337,6 +2460,19 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e return nil } +func (e *executor) translateResults(ctx context.Context, index string, idx *Index, calls []*pql.Call, results []interface{}) (err error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults") + defer span.Finish() + + for i := range results { + results[i], err = e.translateResult(index, idx, calls[i], results[i]) + if err != nil { + return err + } + } + return nil +} + func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { switch result := result.(type) { case *Row: diff --git a/field.go b/field.go index 4189e55a4..b71f9ca7e 100644 --- a/field.go +++ b/field.go @@ -133,7 +133,9 @@ func OptFieldTypeInt(min, max int64) FieldOption { } } -func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption { +// OptFieldTypeTime sets the field type to time. +// Pass true to skip creation of the standard view. +func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) @@ -143,6 +145,7 @@ func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption { } fo.Type = FieldTypeTime fo.TimeQuantum = timeQuantum + fo.NoStandardView = len(opt) >= 1 && opt[0] return nil } } @@ -448,6 +451,7 @@ func (f *Field) loadMeta() error { f.options.Max = pb.Max f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) f.options.Keys = pb.Keys + f.options.NoStandardView = pb.NoStandardView return nil } @@ -514,6 +518,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = 0 f.options.Max = 0 f.options.Keys = opt.Keys + f.options.NoStandardView = opt.NoStandardView // Set the time quantum. if err := f.setTimeQuantum(opt.TimeQuantum); err != nil { f.Close() @@ -797,18 +802,19 @@ func (f *Field) Row(rowID uint64) (*Row, error) { // SetBit sets a bit on a view within the field. func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { viewName := viewStandard + if !f.options.NoStandardView { + // Retrieve view. Exit if it doesn't exist. + view, err := f.createViewIfNotExists(viewName) + if err != nil { + return changed, errors.Wrap(err, "creating view") + } - // Retrieve view. Exit if it doesn't exist. - view, err := f.createViewIfNotExists(viewName) - if err != nil { - return changed, errors.Wrap(err, "creating view") - } - - // Set non-time bit. - if v, err := view.setBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") - } else if v { - changed = v + // Set non-time bit. + if v, err := view.setBit(rowID, colID); err != nil { + return changed, errors.Wrap(err, "setting on view") + } else if v { + changed = v + } } // Exit early if no timestamp is specified. @@ -1092,9 +1098,11 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts standard = []string{viewStandard} } else { standard = viewsByTime(viewStandard, *timestamp, q) - // In order to match the logic of `SetBit()`, we want bits - // with timestamps to write to both time and standard views. - standard = append(standard, viewStandard) + if !f.options.NoStandardView { + // In order to match the logic of `SetBit()`, we want bits + // with timestamps to write to both time and standard views. + standard = append(standard, viewStandard) + } } // Attach bit to each standard view. @@ -1184,9 +1192,10 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return nil } -func (f *Field) importRoaring(data []byte, shard uint64, clear bool) error { - viewName := viewStandard - +func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear bool) error { + if viewName == "" { + viewName = viewStandard + } view, err := f.createViewIfNotExists(viewName) if err != nil { return errors.Wrap(err, "creating view") @@ -1225,13 +1234,14 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FieldOptions represents options to set when initializing a field. type FieldOptions struct { - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` - Keys bool `json:"keys"` - CacheSize uint32 `json:"cacheSize,omitempty"` - CacheType string `json:"cacheType,omitempty"` - Type string `json:"type,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + CacheType string `json:"cacheType,omitempty"` + Type string `json:"type,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // applyDefaultOptions returns a new FieldOptions object @@ -1257,13 +1267,14 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { return nil } return &internal.FieldOptions{ - Type: o.Type, - CacheType: o.CacheType, - CacheSize: o.CacheSize, - Min: o.Min, - Max: o.Max, - TimeQuantum: string(o.TimeQuantum), - Keys: o.Keys, + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + NoStandardView: o.NoStandardView, } } @@ -1295,13 +1306,15 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }) case FieldTypeTime: return json.Marshal(struct { - Type string `json:"type"` - TimeQuantum TimeQuantum `json:"timeQuantum"` - Keys bool `json:"keys"` + Type string `json:"type"` + TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` }{ o.Type, o.TimeQuantum, o.Keys, + o.NoStandardView, }) case FieldTypeMutex: return json.Marshal(struct { diff --git a/fragment.go b/fragment.go index 805112039..028d105b4 100644 --- a/fragment.go +++ b/fragment.go @@ -40,6 +40,7 @@ import ( "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -1492,9 +1493,6 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor lastRowID = rowID rowSet[rowID] = struct{}{} } - - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) } f.mu.Lock() @@ -1518,6 +1516,9 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor // Update cache counts for all affected rows. for rowID := range rowSet { + // Invalidate block checksum. + delete(f.checksums, int(rowID/HashBlockSize)) + n := results.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } @@ -1734,7 +1735,6 @@ func (f *fragment) snapshot() error { // f.mu must be locked when calling it. func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) @@ -2188,6 +2188,9 @@ func (s *fragmentSyncer) isClosing() bool { // syncFragment compares checksums for the local and remote fragments and // then merges any blocks which have differences. func (s *fragmentSyncer) syncFragment() error { + span, ctx := tracing.StartSpanFromContext(context.Background(), "FragmentSyncer.syncFragment") + defer span.Finish() + // Determine replica set. nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard) if len(nodes) == 1 { @@ -2205,7 +2208,7 @@ func (s *fragmentSyncer) syncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.view, s.Fragment.shard) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(ctx, &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.view, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -2265,6 +2268,9 @@ func (s *fragmentSyncer) syncFragment() error { // syncBlock sends and receives all rows for a given block. // Returns an error if any remote hosts are unreachable. func (s *fragmentSyncer) syncBlock(id int) error { + span, ctx := tracing.StartSpanFromContext(context.Background(), "FragmentSyncer.syncBlock") + defer span.Finish() + f := s.Fragment // Read pairs from each remote block. @@ -2284,7 +2290,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.view, f.shard, id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(ctx, &node.URI, f.index, f.field, f.view, f.shard, id) if err != nil { return errors.Wrap(err, "getting block") } @@ -2346,7 +2352,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(ctx, uris[i], f.index, queryRequest) if err != nil { return errors.Wrap(err, "executing") } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 4e3007957..b36bf3380 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -25,6 +25,8 @@ import ( "testing" "testing/quick" + "golang.org/x/sync/errgroup" + "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" @@ -1399,6 +1401,21 @@ func TestFragment_ImportSet(t *testing.T) { } } +func TestFragment_ConcurrentImport(t *testing.T) { + t.Run("bulkImportStandard", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + eg := errgroup.Group{} + eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) + eg.Go(func() error { return f.bulkImportStandard([]uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) + err := eg.Wait() + if err != nil { + t.Fatalf("importing data to fragment: %v", err) + } + }) +} + // Ensure a fragment can import mutually exclusive values. func TestFragment_ImportMutex(t *testing.T) { tests := []struct { diff --git a/handler.go b/handler.go index 9fc3af368..8fe51dc4c 100644 --- a/handler.go +++ b/handler.go @@ -96,6 +96,11 @@ type ImportRequest struct { Timestamps []int64 } +type ImportRoaringRequest struct { + Clear bool + Views map[string][]byte +} + type ImportResponse struct { Err string } diff --git a/holder.go b/holder.go index 68643915e..9e58e8057 100644 --- a/holder.go +++ b/holder.go @@ -30,6 +30,7 @@ import ( "github.com/pilosa/pilosa/logger" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -57,7 +58,7 @@ type Holder struct { NewPrimaryTranslateStore func(interface{}) TranslateStore // opened channel is closed once Open() completes. - opened chan struct{} + opened lockedChan broadcaster broadcaster @@ -79,13 +80,39 @@ type Holder struct { Logger logger.Logger } +// lockedChan looks a little ridiculous admittedly, but exists for good reason. +// The channel within is used (for example) to signal to other goroutines when +// the Holder has finished opening (via closing the channel). However, it is +// possible for the holder to be closed and then reopened, but a channel which +// is closed cannot be re-opened. We must create a new channel - this creates a +// data race with any goroutine which might be accessing the channel. To ensure +// that there is no data race on the value of the channel itself, we wrap any +// operation on it with an RWMutex so that we can guarantee that nothing is +// trying to listen on it when it gets swapped. +type lockedChan struct { + ch chan struct{} + mu sync.RWMutex +} + +func (lc *lockedChan) Close() { + lc.mu.RLock() + close(lc.ch) + lc.mu.RUnlock() +} + +func (lc *lockedChan) Recv() { + lc.mu.RLock() + <-lc.ch + lc.mu.RUnlock() +} + // NewHolder returns a new instance of Holder. func NewHolder() *Holder { return &Holder{ indexes: make(map[string]*Index), closing: make(chan struct{}), - opened: make(chan struct{}), + opened: lockedChan{ch: make(chan struct{})}, translateFile: NewTranslateFile(), NewPrimaryTranslateStore: newNopTranslateStore, @@ -159,7 +186,7 @@ func (h *Holder) Open() error { h.Stats.Open() - close(h.opened) + h.opened.Close() return nil } @@ -184,7 +211,9 @@ func (h *Holder) Close() error { } // Reset opened in case Holder needs to be reopened. - h.opened = make(chan struct{}) + h.opened.mu.Lock() + h.opened.ch = make(chan struct{}) + h.opened.mu.Unlock() return nil } @@ -474,7 +503,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath()) + h.Logger.Printf("ERROR flushing cache: err=%s, path=%s", err, fragment.cachePath()) } } } @@ -535,7 +564,7 @@ func (h *Holder) setFileLimit() { h.Logger.Printf("ERROR checking open file limit: %s", err) } else { if oldLimit.Cur < fileLimit { - h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit) + h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/latest/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit) } } } @@ -692,6 +721,9 @@ func (s *holderSyncer) SyncHolder() error { // syncIndex synchronizes index attributes with the rest of the cluster. func (s *holderSyncer) syncIndex(index string) error { + span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncIndex") + defer span.Finish() + // Retrieve index reference. idx := s.Holder.Index(index) if idx == nil { @@ -710,7 +742,7 @@ func (s *holderSyncer) syncIndex(index string) error { for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks) + m, err := s.Cluster.InternalClient.ColumnAttrDiff(ctx, &node.URI, index, blks) if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { @@ -735,6 +767,9 @@ func (s *holderSyncer) syncIndex(index string) error { // syncField synchronizes field attributes with the rest of the cluster. func (s *holderSyncer) syncField(index, name string) error { + span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncField") + defer span.Finish() + // Retrieve field reference. f := s.Holder.Field(index, name) if f == nil { @@ -754,7 +789,7 @@ func (s *holderSyncer) syncField(index, name string) error { for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks) + m, err := s.Cluster.InternalClient.RowAttrDiff(ctx, &node.URI, index, name, blks) if err == ErrFieldNotFound { continue // field not created remotely yet, skip } else if err != nil { diff --git a/http/client.go b/http/client.go index 0d6df281f..b18757a46 100644 --- a/http/client.go +++ b/http/client.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/encoding/proto" + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -66,6 +67,8 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) // MaxShardByIndex returns the number of shards on a server by index. func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.MaxShardByIndex") + defer span.Finish() return c.maxShardByIndex(ctx) } @@ -91,9 +94,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 defer resp.Body.Close() var rsp getShardsMaxResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, fmt.Errorf("json decode: %s", err) } @@ -102,6 +103,9 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 // Schema returns all index and field schema information. func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema") + defer span.Finish() + // Execute request against the host. u := c.defaultURI.Path("/schema") @@ -130,6 +134,9 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error // CreateIndex creates a new index on the server. func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex") + defer span.Finish() + // Encode query request. buf, err := json.Marshal(&postIndexRequest{ Options: opt, @@ -152,7 +159,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - if resp.StatusCode == http.StatusConflict { + if resp != nil && resp.StatusCode == http.StatusConflict { return pilosa.ErrIndexExists } return err @@ -162,6 +169,9 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo // FragmentNodes returns a list of nodes that own a shard. func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentNodes") + defer span.Finish() + // Execute request against the host. u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes") u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode() @@ -191,6 +201,9 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard // Nodes returns a list of all nodes. func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Nodes") + defer span.Finish() + // Execute request against the host. u := uriPathToURL(c.defaultURI, "/internal/nodes") @@ -219,11 +232,16 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) { // Query executes query against the index. func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Query") + defer span.Finish() return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode") + defer span.Finish() + if index == "" { return nil, pilosa.ErrIndexRequired } else if queryRequest.Query == "" { @@ -258,8 +276,6 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) } qresp := &pilosa.QueryResponse{} @@ -274,6 +290,9 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s // Import bulk imports bits for a single shard to a host. func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit, opts ...pilosa.ImportOption) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -321,6 +340,9 @@ func getCoordinatorNode(nodes []*pilosa.Node) *pilosa.Node { // ImportK bulk imports bits specified by string keys to a host. func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits []pilosa.Bit, opts ...pilosa.ImportOption) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportK") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -361,6 +383,9 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits } func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureIndex") + defer span.Finish() + err := c.CreateIndex(ctx, name, options) if err == nil || errors.Cause(err) == pilosa.ErrIndexExists { return nil @@ -369,10 +394,14 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p } func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureField") + defer span.Finish() return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) } func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureFieldWithOptions") + defer span.Finish() err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { return nil @@ -408,6 +437,9 @@ func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, // importNode sends a pre-marshaled import request to a node. func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode") + defer span.Finish() + // Create URL & HTTP request. path := fmt.Sprintf("/index/%s/field/%s/import", index, field) u := nodePathToURL(node, path) @@ -455,6 +487,9 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde // ImportValue bulk imports field values for a single shard to a host. func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValue") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -493,6 +528,9 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s // ImportValueK bulk imports keyed field values to a host. func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK") + defer span.Finish() + buf, err := c.marshalImportValuePayload(index, field, 0, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) @@ -550,7 +588,10 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, data []byte, opts ...pilosa.ImportOption) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -560,32 +601,27 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind uri = c.defaultURI } - // Set up import options. - options := &pilosa.ImportOptions{} - for _, opt := range opts { - err := opt(options) - if err != nil { - return errors.Wrap(err, "applying option") - } - } - vals := url.Values{} vals.Set("remote", strconv.FormatBool(remote)) - if options.Clear { - vals.Set("clear", "true") - } url := fmt.Sprintf("%s/index/%s/field/%s/import-roaring/%d?%s", uri, index, field, shard, vals.Encode()) + // Marshal data to protobuf. + data, err := c.serializer.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshal import request") + } + // Generate HTTP request. - req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) if err != nil { return errors.Wrap(err, "creating request") } - req.Header.Set("Content-Type", "application/x-binary") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.executeRequest(req.WithContext(ctx)) + resp, err := c.executeRequest(httpReq.WithContext(ctx)) if err != nil { return err } @@ -595,13 +631,16 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind rbody := &pilosa.ImportResponse{} dec.Decode(rbody) if rbody.Err != "" { - return errors.Errorf("importing roaring: %v", rbody.Err) + return errors.Wrap(errors.New(rbody.Err), "importing roaring") } return nil } // ExportCSV bulk exports data for a single shard from a host to CSV format. func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -632,6 +671,9 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha // exportNode copies a CSV export from a node to w. func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.exportNodeCSV") + defer span.Finish() + // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ @@ -664,6 +706,9 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i } func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI") + defer span.Finish() + node := &pilosa.Node{ URI: uri, } @@ -671,6 +716,9 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field } func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.backupShardNode") + defer span.Finish() + u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, @@ -689,7 +737,7 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - if resp.StatusCode == http.StatusNotFound { + if resp != nil && resp.StatusCode == http.StatusNotFound { return nil, pilosa.ErrFragmentNotFound } return nil, err @@ -699,11 +747,16 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin } func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateField") + defer span.Finish() return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) } // CreateField creates a new field on the server. func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions") + defer span.Finish() + if index == "" { return pilosa.ErrIndexRequired } @@ -746,7 +799,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - if resp.StatusCode == http.StatusConflict { + if resp != nil && resp.StatusCode == http.StatusConflict { return pilosa.ErrFieldExists } return err @@ -758,6 +811,9 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks") + defer span.Finish() + if uri == nil { uri = c.defaultURI } @@ -782,7 +838,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { // Return the appropriate error. - if resp.StatusCode == http.StatusNotFound { + if resp != nil && resp.StatusCode == http.StatusNotFound { return nil, pilosa.ErrFragmentNotFound } return nil, err @@ -799,6 +855,9 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in // BlockData returns row/column id pairs for a block. func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.BlockData") + defer span.Finish() + if uri == nil { panic("need to pass a URI to BlockData") } @@ -825,7 +884,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - if resp.StatusCode == http.StatusNotFound { + if resp != nil && resp.StatusCode == http.StatusNotFound { return nil, nil, nil } return nil, nil, err @@ -844,6 +903,9 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, // ColumnAttrDiff returns data from differing blocks on a remote host. func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ColumnAttrDiff") + defer span.Finish() + if uri == nil { uri = c.defaultURI } @@ -881,6 +943,9 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in // RowAttrDiff returns data from differing blocks on a remote host. func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RowAttrDiff") + defer span.Finish() + if uri == nil { uri = c.defaultURI } @@ -904,7 +969,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - if resp.StatusCode == http.StatusNotFound { + if resp != nil && resp.StatusCode == http.StatusNotFound { return nil, pilosa.ErrFieldNotFound } return nil, err @@ -921,6 +986,9 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index // SendMessage posts a message synchronously. func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.SendMessage") + defer span.Finish() + u := uriPathToURL(uri, "/internal/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) if err != nil { @@ -937,6 +1005,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ // executeRequest executes the given request and checks the Response func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) { + tracing.GlobalTracer.InjectHTTPHeaders(req) resp, err := c.httpClient.Do(req) if err != nil { return nil, errors.Wrap(err, "executing request") diff --git a/http/client_test.go b/http/client_test.go index 23d8b8573..fee3754df 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -408,8 +408,9 @@ func TestClient_ImportRoaring(t *testing.T) { // Send import request. host := cluster[0].URL() c := MustNewClient(host, http.GetHTTPClient(nil)) - roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringData); err != nil { + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -432,8 +433,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ := hex.DecodeString("3A30000001000000010001001000000003000400") // [65539, 65540] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [65539, 65540] + roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -454,8 +456,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ = hex.DecodeString("3A300000020000000000010001000100180000001C0000000400060001000300") // [4, 6, 65537, 65539] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [4, 6, 65537, 65539] + roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -476,8 +479,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ = hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -981,3 +985,13 @@ func MustNewClient(host string, h *gohttp.Client) *Client { } return &Client{InternalClient: c} } + +func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaringRequest { + roaringData, _ := hex.DecodeString(viewData) + return &pilosa.ImportRoaringRequest{ + Clear: clear, + Views: map[string][]byte{ + "": roaringData, + }, + } +} diff --git a/http/handler.go b/http/handler.go index 1ecbd2fb1..d728bfcb5 100644 --- a/http/handler.go +++ b/http/handler.go @@ -24,9 +24,8 @@ import ( "io/ioutil" "net" "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" + "net/url" // Imported for its side-effect of registering pprof endpoints with the server. "reflect" "runtime/debug" "strconv" @@ -37,7 +36,7 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/logger" - + "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -222,6 +221,15 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { }) } +func (h *Handler) extractTracing(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + span, ctx := tracing.GlobalTracer.ExtractHTTPHeaders(r) + defer span.Finish() + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() @@ -262,6 +270,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData") router.Use(handler.queryArgValidator) + router.Use(handler.extractTracing) return router } @@ -707,7 +716,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { case pilosa.FieldTypeInt: fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) case pilosa.FieldTypeTime: - fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) + fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView)) case pilosa.FieldTypeMutex: fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeBool: @@ -730,13 +739,14 @@ type postFieldRequest struct { // fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, // and used for input validation. type fieldOptions struct { - Type string `json:"type,omitempty"` - CacheType *string `json:"cacheType,omitempty"` - CacheSize *uint32 `json:"cacheSize,omitempty"` - Min *int64 `json:"min,omitempty"` - Max *int64 `json:"max,omitempty"` - TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` - Keys *bool `json:"keys,omitempty"` + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` + TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` + NoStandardView bool `json:"noStandardView,omitempty"` } func (o *fieldOptions) validate() error { @@ -1497,10 +1507,18 @@ func GetHTTPClient(t *tls.Config) *http.Client { // handlPostRoaringImport func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Content-Type") != "application/x-binary" { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + q := r.URL.Query() remoteStr := q.Get("remote") var remote bool @@ -1508,9 +1526,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request remote = true } - // If the clear flag is true, treat the import as clear bits. - doClear := q.Get("clear") == "true" - // Read entire body. body, err := ioutil.ReadAll(r.Body) if err != nil { @@ -1518,6 +1533,12 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } + req := &pilosa.ImportRoaringRequest{} + if err := h.api.Serializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + urlVars := mux.Vars(r) shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) if err != nil { @@ -1527,7 +1548,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body, pilosa.OptImportOptionsClear(doClear)) + err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { diff --git a/index_test.go b/index_test.go index bc412646a..21f9a91d9 100644 --- a/index_test.go +++ b/index_test.go @@ -70,6 +70,22 @@ func TestIndex_CreateField(t *testing.T) { }) }) + // Ensure time quantum can be set appropriately on a new field. + t.Run("TimeQuantumNoStandardView", func(t *testing.T) { + t.Run("Explicit", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + + // Create field with explicit quantum with no standard view + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), true)) + if err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected field time quantum: %s", q) + } + }) + }) + // Ensure field can include range columns. t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { diff --git a/internal/private.pb.go b/internal/private.pb.go index 200aa4bed..0c5862b13 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{0} + return fileDescriptor_private_08d4c0c27f7a355f, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -83,6 +83,7 @@ type FieldOptions struct { Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` + NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -92,7 +93,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{1} + return fileDescriptor_private_08d4c0c27f7a355f, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -170,6 +171,13 @@ func (m *FieldOptions) GetKeys() bool { return false } +func (m *FieldOptions) GetNoStandardView() bool { + if m != nil { + return m.NoStandardView + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -181,7 +189,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{2} + return fileDescriptor_private_08d4c0c27f7a355f, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -232,7 +240,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{3} + return fileDescriptor_private_08d4c0c27f7a355f, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -308,7 +316,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{4} + return fileDescriptor_private_08d4c0c27f7a355f, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -362,7 +370,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{5} + return fileDescriptor_private_08d4c0c27f7a355f, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -409,7 +417,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{6} + return fileDescriptor_private_08d4c0c27f7a355f, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -458,7 +466,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{7} + return fileDescriptor_private_08d4c0c27f7a355f, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -519,7 +527,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{8} + return fileDescriptor_private_08d4c0c27f7a355f, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -567,7 +575,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{9} + return fileDescriptor_private_08d4c0c27f7a355f, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -623,7 +631,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{10} + return fileDescriptor_private_08d4c0c27f7a355f, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -685,7 +693,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{11} + return fileDescriptor_private_08d4c0c27f7a355f, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -741,7 +749,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{12} + return fileDescriptor_private_08d4c0c27f7a355f, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -804,7 +812,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{13} + return fileDescriptor_private_08d4c0c27f7a355f, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -865,7 +873,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{14} + return fileDescriptor_private_08d4c0c27f7a355f, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -913,7 +921,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{15} + return fileDescriptor_private_08d4c0c27f7a355f, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -969,7 +977,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{16} + return fileDescriptor_private_08d4c0c27f7a355f, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1033,7 +1041,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{17} + return fileDescriptor_private_08d4c0c27f7a355f, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1102,7 +1110,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{18} + return fileDescriptor_private_08d4c0c27f7a355f, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1157,7 +1165,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{19} + return fileDescriptor_private_08d4c0c27f7a355f, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1213,7 +1221,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{20} + return fileDescriptor_private_08d4c0c27f7a355f, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1275,7 +1283,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{21} + return fileDescriptor_private_08d4c0c27f7a355f, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1330,7 +1338,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{22} + return fileDescriptor_private_08d4c0c27f7a355f, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1386,7 +1394,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{23} + return fileDescriptor_private_08d4c0c27f7a355f, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1450,7 +1458,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{24} + return fileDescriptor_private_08d4c0c27f7a355f, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1520,7 +1528,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{25} + return fileDescriptor_private_08d4c0c27f7a355f, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1583,7 +1591,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{26} + return fileDescriptor_private_08d4c0c27f7a355f, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1649,7 +1657,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{27} + return fileDescriptor_private_08d4c0c27f7a355f, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1735,7 +1743,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{28} + return fileDescriptor_private_08d4c0c27f7a355f, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1812,7 +1820,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{29} + return fileDescriptor_private_08d4c0c27f7a355f, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1873,7 +1881,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{30} + return fileDescriptor_private_08d4c0c27f7a355f, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1920,7 +1928,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{31} + return fileDescriptor_private_08d4c0c27f7a355f, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1968,7 +1976,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{32} + return fileDescriptor_private_08d4c0c27f7a355f, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2021,7 +2029,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_725d4d7695f6ae76, []int{33} + return fileDescriptor_private_08d4c0c27f7a355f, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2186,6 +2194,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.NoStandardView { + dAtA[i] = 0x60 + i++ + if m.NoStandardView { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -3556,6 +3574,9 @@ func (m *FieldOptions) Size() (n int) { if m.Keys { n += 2 } + if m.NoStandardView { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4579,6 +4600,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } } m.Keys = bool(v != 0) + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NoStandardView", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.NoStandardView = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8836,79 +8877,79 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_725d4d7695f6ae76) } +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_08d4c0c27f7a355f) } -var fileDescriptor_private_725d4d7695f6ae76 = []byte{ - // 1121 bytes of a gzipped FileDescriptorProto +var fileDescriptor_private_08d4c0c27f7a355f = []byte{ + // 1131 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5, 0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51, 0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c, 0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7, - 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66, - 0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74, - 0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29, - 0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c, - 0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e, - 0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b, - 0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1, - 0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76, - 0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d, - 0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34, - 0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88, - 0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91, - 0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7, - 0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82, - 0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42, - 0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8, - 0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43, - 0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa, - 0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77, - 0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce, - 0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a, - 0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, - 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03, - 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75, - 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, - 0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21, - 0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f, - 0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf, - 0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e, - 0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58, - 0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce, - 0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f, - 0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14, - 0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34, - 0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4, - 0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14, - 0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b, - 0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28, - 0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99, - 0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34, - 0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a, - 0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91, - 0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24, - 0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19, - 0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e, - 0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0, - 0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4, - 0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70, - 0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1, - 0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56, - 0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43, - 0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca, - 0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68, - 0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb, - 0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a, - 0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0, - 0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a, - 0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34, - 0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17, - 0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2, - 0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84, - 0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26, - 0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1, - 0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75, - 0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc, - 0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00, - 0x00, + 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe2, 0x92, 0x47, 0xa8, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xb3, 0xbb, + 0x76, 0x1c, 0x12, 0x05, 0xee, 0xe6, 0xfc, 0xce, 0x99, 0xf3, 0x7d, 0xce, 0xec, 0x42, 0x77, 0x9c, + 0xc5, 0x27, 0x4c, 0xf2, 0x8d, 0x71, 0x26, 0xa4, 0x08, 0x5a, 0x71, 0x2a, 0x79, 0x96, 0xb2, 0x84, + 0x3c, 0x87, 0x76, 0x94, 0x0e, 0xf9, 0xd9, 0x0e, 0x97, 0x2c, 0x08, 0xc0, 0x7f, 0xc1, 0x27, 0x79, + 0xe8, 0xad, 0x39, 0xeb, 0x2d, 0x8a, 0xe7, 0xe0, 0x03, 0x58, 0xdc, 0xcb, 0xd8, 0xe0, 0x78, 0xfb, + 0x2c, 0xce, 0x25, 0x4f, 0x07, 0x3c, 0xf4, 0x91, 0x3b, 0x83, 0x92, 0x37, 0x0e, 0xdc, 0xfa, 0x2a, + 0xe6, 0xc9, 0xf0, 0xbb, 0xb1, 0x8c, 0x45, 0x9a, 0x07, 0xef, 0x42, 0x7b, 0x8b, 0x0d, 0x8e, 0xf8, + 0xde, 0x64, 0xcc, 0x51, 0x63, 0x9b, 0x56, 0x40, 0xc9, 0xed, 0xc7, 0xaf, 0xb5, 0xc6, 0x2e, 0xad, + 0x80, 0x60, 0x0d, 0x3a, 0x7b, 0xf1, 0x88, 0x7f, 0x5f, 0xb0, 0x54, 0x16, 0xa3, 0xb0, 0x81, 0xb7, + 0xeb, 0x90, 0x72, 0x15, 0x15, 0xb7, 0x90, 0x85, 0xe7, 0x60, 0x09, 0xbc, 0x9d, 0x38, 0x0d, 0xdb, + 0x6b, 0xce, 0xba, 0x47, 0xd5, 0x11, 0x11, 0x76, 0x16, 0x82, 0x41, 0xd8, 0x59, 0x19, 0x62, 0x67, + 0x3a, 0xc4, 0x5d, 0xd1, 0x97, 0x2c, 0x1d, 0xb2, 0x6c, 0x78, 0x10, 0xf3, 0xd3, 0xf0, 0x96, 0x0e, + 0x71, 0x1a, 0x25, 0x04, 0x16, 0xa3, 0xd1, 0x58, 0x64, 0x92, 0xf2, 0x7c, 0x2c, 0xd2, 0x1c, 0x2d, + 0x6e, 0x67, 0x59, 0xe8, 0xa0, 0x13, 0xea, 0x48, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0x0c, 0x8e, 0x7b, + 0x4c, 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0xff, 0x43, 0x03, 0x73, 0x6c, 0xe4, 0x34, 0xa1, + 0x50, 0xcc, 0x57, 0xe8, 0x6a, 0x14, 0x09, 0x85, 0xe2, 0x7d, 0xcc, 0x98, 0x4f, 0x35, 0xa1, 0xd0, + 0xfe, 0x11, 0xcb, 0x86, 0x98, 0x29, 0x9f, 0x6a, 0x42, 0xc5, 0x82, 0xde, 0xea, 0xf4, 0xe0, 0x99, + 0x44, 0xb0, 0x5c, 0xb3, 0x6f, 0xdc, 0x5c, 0x81, 0x26, 0x15, 0xa7, 0x51, 0x2f, 0x0f, 0x9d, 0x35, + 0x6f, 0xdd, 0xa7, 0x86, 0xc2, 0x22, 0x88, 0xa4, 0x18, 0xa5, 0x8a, 0xe5, 0x22, 0xab, 0x02, 0xc8, + 0x3d, 0x68, 0x60, 0x45, 0x54, 0x94, 0xd5, 0x5d, 0x75, 0x24, 0x3f, 0x3b, 0xd0, 0xde, 0x61, 0x67, + 0xe8, 0x46, 0x1e, 0x3c, 0x85, 0x96, 0xcd, 0x13, 0x0a, 0x75, 0x1e, 0xbf, 0xbf, 0x61, 0x1b, 0x6c, + 0xa3, 0x14, 0xdb, 0xb0, 0x32, 0xdb, 0xa9, 0xcc, 0x26, 0xb4, 0xbc, 0xf2, 0xf6, 0xe7, 0xd0, 0x9d, + 0x62, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, + 0x73, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x03, 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, + 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x79, 0xc6, 0x75, 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, + 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, 0x37, 0xd3, 0xf1, 0x0f, 0x7a, 0x49, + 0xdf, 0xfa, 0x70, 0xb5, 0x6c, 0xf0, 0x00, 0x7c, 0x35, 0x6a, 0xe8, 0x42, 0xe7, 0xf1, 0x9d, 0x2a, + 0x4f, 0xe5, 0x14, 0x52, 0x14, 0x20, 0x89, 0x55, 0x8a, 0xfe, 0x5c, 0x19, 0xd8, 0x9c, 0x56, 0x7a, + 0x68, 0x4c, 0x79, 0x68, 0x6a, 0xa5, 0x32, 0x55, 0x1f, 0x53, 0x63, 0xed, 0x99, 0x0d, 0xf7, 0xa6, + 0xd6, 0xc8, 0x00, 0xde, 0xd1, 0x1a, 0xbe, 0x3c, 0x61, 0x71, 0xc2, 0x0e, 0x93, 0x6b, 0x56, 0x64, + 0x8e, 0xe3, 0x21, 0x2c, 0xe0, 0xdd, 0xa8, 0x67, 0xa6, 0xc0, 0x92, 0xe4, 0x07, 0x23, 0xaf, 0x5a, + 0x7f, 0x97, 0x8d, 0xb8, 0xd1, 0x86, 0xe7, 0x32, 0x5e, 0xf7, 0xea, 0x78, 0x95, 0x61, 0x35, 0x2e, + 0x6a, 0xd5, 0x79, 0xca, 0x30, 0x12, 0xe4, 0x09, 0x34, 0xfb, 0x83, 0x23, 0x3e, 0x62, 0xc1, 0x87, + 0xb0, 0x80, 0x1e, 0xf2, 0xdc, 0x74, 0xf4, 0xed, 0x99, 0x4a, 0x51, 0xcb, 0x27, 0x3d, 0x13, 0xd9, + 0x5c, 0x9f, 0x1e, 0x40, 0x13, 0xad, 0xe7, 0xa1, 0x3f, 0xab, 0x06, 0x71, 0x6a, 0xd8, 0x64, 0x1b, + 0xbc, 0x7d, 0x1a, 0xa9, 0x49, 0x45, 0x0f, 0xac, 0x16, 0x43, 0x29, 0xdd, 0x5f, 0x8b, 0x5c, 0x9a, + 0x3c, 0xe1, 0x59, 0x61, 0x2f, 0x45, 0x26, 0x31, 0x47, 0x5d, 0x8a, 0x67, 0x92, 0x83, 0xbf, 0x2b, + 0x86, 0x3c, 0x58, 0x04, 0x37, 0xea, 0x19, 0x1d, 0x6e, 0xd4, 0x0b, 0xde, 0x43, 0xf5, 0x26, 0x35, + 0xdd, 0xca, 0x89, 0x7d, 0x1a, 0x51, 0x34, 0x7c, 0x1f, 0xba, 0x51, 0xbe, 0x25, 0x44, 0x36, 0x8c, + 0x53, 0x26, 0x45, 0x66, 0xde, 0x80, 0x69, 0x10, 0x27, 0x48, 0x32, 0xa9, 0x37, 0x76, 0x9b, 0x6a, + 0x82, 0x3c, 0x83, 0x25, 0x65, 0x14, 0x09, 0x5b, 0xef, 0x15, 0x68, 0x2a, 0xac, 0x74, 0xc2, 0x50, + 0x95, 0x06, 0xb7, 0xae, 0xe1, 0x5b, 0xad, 0x61, 0xfb, 0x84, 0xa7, 0xb2, 0xd6, 0x31, 0x48, 0xa3, + 0x82, 0x2e, 0xd5, 0x44, 0x40, 0x74, 0x80, 0x26, 0x92, 0xc5, 0x2a, 0x12, 0x85, 0x52, 0xe4, 0x91, + 0x5f, 0x1d, 0x00, 0xeb, 0x50, 0x91, 0x97, 0x57, 0x9c, 0xcb, 0xaf, 0x04, 0xeb, 0xb6, 0xf2, 0x66, + 0x5a, 0x96, 0x2a, 0x29, 0x8d, 0x53, 0xdb, 0x19, 0x1f, 0x57, 0x9d, 0xa1, 0x4b, 0x7a, 0x77, 0xa6, + 0x33, 0xb4, 0xd5, 0xaa, 0x3f, 0x5e, 0x42, 0xa7, 0x86, 0xcf, 0xed, 0x92, 0x8f, 0xca, 0x2e, 0x71, + 0x67, 0x55, 0x22, 0x6e, 0x54, 0xda, 0x5e, 0x79, 0x01, 0x9d, 0x1a, 0x3c, 0x57, 0xe3, 0x3a, 0xdc, + 0x9e, 0x9e, 0x43, 0xbb, 0xdf, 0x67, 0x61, 0x12, 0x43, 0x77, 0x2b, 0x29, 0x72, 0xc9, 0x33, 0xa3, + 0x4e, 0x3d, 0x0a, 0x1a, 0x28, 0x8b, 0x57, 0x01, 0xf3, 0xeb, 0x17, 0xdc, 0x87, 0x86, 0x4a, 0xa3, + 0x1e, 0xa7, 0x8b, 0x39, 0xd6, 0x4c, 0x72, 0x00, 0xad, 0xcd, 0x7e, 0xf4, 0x3c, 0x13, 0xc5, 0x78, + 0xae, 0xd3, 0xf6, 0x4d, 0x77, 0x2f, 0xbe, 0xe9, 0xde, 0x85, 0x37, 0xdd, 0x2f, 0xdf, 0x74, 0xd2, + 0x87, 0x65, 0xbd, 0x2a, 0xd5, 0x14, 0xdf, 0x64, 0xe1, 0xd8, 0x87, 0xd4, 0xab, 0x3d, 0xa4, 0x7d, + 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x54, 0xfa, 0xbb, 0x0b, 0xcb, 0x94, 0xe7, 0xf1, 0x6b, 0x1e, 0xa5, + 0xb9, 0xcc, 0x8a, 0x81, 0xda, 0x49, 0xea, 0xfe, 0x37, 0xe2, 0xd0, 0x64, 0xdb, 0xa3, 0x9a, 0xb8, + 0x4e, 0xa7, 0x07, 0x8f, 0xa0, 0x33, 0x3b, 0xb3, 0x17, 0x45, 0xeb, 0x22, 0xc1, 0x23, 0x58, 0xe8, + 0x8b, 0x22, 0x1b, 0x94, 0xed, 0x5b, 0xdb, 0x93, 0xda, 0x33, 0xcd, 0xa6, 0x56, 0xac, 0x36, 0x1a, + 0x8d, 0x2b, 0x46, 0xe3, 0xe9, 0x4c, 0x2b, 0x85, 0x4d, 0xbc, 0xf0, 0x56, 0x75, 0x61, 0x8a, 0x4d, + 0xa7, 0xa5, 0xc9, 0x2f, 0x0e, 0xdc, 0xaa, 0xbb, 0x70, 0xad, 0xc1, 0x2d, 0x2b, 0xe2, 0xce, 0xad, + 0x88, 0x37, 0xaf, 0x22, 0x7e, 0x55, 0x91, 0xea, 0x9b, 0xa0, 0x51, 0xfb, 0x26, 0x20, 0xc7, 0x70, + 0xef, 0x42, 0x99, 0xb6, 0xc4, 0x68, 0xac, 0xfa, 0xe1, 0x5f, 0x94, 0x4b, 0xad, 0xb4, 0x2c, 0x33, + 0x85, 0x6a, 0x53, 0x4d, 0x90, 0x4f, 0xe1, 0x6e, 0x9f, 0xcb, 0x5a, 0x91, 0x6c, 0xb7, 0xad, 0x81, + 0xb7, 0xcb, 0x4f, 0x2f, 0x09, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xee, 0x8f, 0x87, 0x4c, 0xf2, 0x1b, + 0xdd, 0xde, 0x84, 0xd6, 0x9e, 0x18, 0x8b, 0x44, 0xbc, 0x9a, 0x5c, 0x31, 0xf5, 0x21, 0x2c, 0xe8, + 0xfd, 0xad, 0xd7, 0x48, 0x9b, 0x5a, 0x92, 0xdc, 0x51, 0x0d, 0x3d, 0x60, 0xc9, 0xa0, 0x48, 0x94, + 0x1b, 0xea, 0x7b, 0x31, 0xdf, 0x5c, 0xfa, 0xe3, 0x7c, 0xd5, 0xf9, 0xf3, 0x7c, 0xd5, 0x79, 0x73, + 0xbe, 0xea, 0xfc, 0xf6, 0xd7, 0xea, 0xff, 0x0e, 0x9b, 0xf8, 0xdf, 0xf1, 0xe4, 0xef, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x61, 0x80, 0xe4, 0xef, 0x88, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 2e484b037..1cd88c82c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -15,6 +15,7 @@ message FieldOptions { int64 Max = 10; string TimeQuantum = 5; bool Keys = 11; + bool NoStandardView = 12; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 8d78db985..34ac4db9f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{0} + return fileDescriptor_public_5eafd62083455670, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{1} + return fileDescriptor_public_5eafd62083455670, []int{1} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{2} + return fileDescriptor_public_5eafd62083455670, []int{2} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -215,7 +215,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{3} + return fileDescriptor_public_5eafd62083455670, []int{3} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -270,7 +270,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{4} + return fileDescriptor_public_5eafd62083455670, []int{4} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,7 +325,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{5} + return fileDescriptor_public_5eafd62083455670, []int{5} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -381,7 +381,7 @@ func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{6} + return fileDescriptor_public_5eafd62083455670, []int{6} } func (m *Bit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -444,7 +444,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{7} + return fileDescriptor_public_5eafd62083455670, []int{7} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +510,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{8} + return fileDescriptor_public_5eafd62083455670, []int{8} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +592,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{9} + return fileDescriptor_public_5eafd62083455670, []int{9} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -644,7 +644,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{10} + return fileDescriptor_public_5eafd62083455670, []int{10} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,7 +728,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{11} + return fileDescriptor_public_5eafd62083455670, []int{11} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -797,7 +797,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{12} + return fileDescriptor_public_5eafd62083455670, []int{12} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,7 +907,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{13} + return fileDescriptor_public_5eafd62083455670, []int{13} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1008,7 +1008,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{14} + return fileDescriptor_public_5eafd62083455670, []int{14} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1079,6 +1079,116 @@ func (m *ImportValueRequest) GetValues() []int64 { return nil } +type ImportRoaringRequestView struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } +func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequestView) ProtoMessage() {} +func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { + return fileDescriptor_public_5eafd62083455670, []int{15} +} +func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) +} +func (m *ImportRoaringRequestView) XXX_Size() int { + return m.Size() +} +func (m *ImportRoaringRequestView) XXX_DiscardUnknown() { + xxx_messageInfo_ImportRoaringRequestView.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportRoaringRequestView proto.InternalMessageInfo + +func (m *ImportRoaringRequestView) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ImportRoaringRequestView) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type ImportRoaringRequest struct { + Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } +func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequest) ProtoMessage() {} +func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_public_5eafd62083455670, []int{16} +} +func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) +} +func (m *ImportRoaringRequest) XXX_Size() int { + return m.Size() +} +func (m *ImportRoaringRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ImportRoaringRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportRoaringRequest proto.InternalMessageInfo + +func (m *ImportRoaringRequest) GetClear() bool { + if m != nil { + return m.Clear + } + return false +} + +func (m *ImportRoaringRequest) GetViews() []*ImportRoaringRequestView { + if m != nil { + return m.Views + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") @@ -1095,6 +1205,8 @@ func init() { proto.RegisterType((*QueryResult)(nil), "internal.QueryResult") proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") + proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") + proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") } func (m *Row) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -1979,6 +2091,82 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Data) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Clear { + dAtA[i] = 0x8 + i++ + if m.Clear { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.Views) > 0 { + for _, msg := range m.Views { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2434,6 +2622,47 @@ func (m *ImportValueRequest) Size() (n int) { return n } +func (m *ImportRoaringRequestView) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ImportRoaringRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Clear { + n += 2 + } + if len(m.Views) > 0 { + for _, e := range m.Views { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { for { n++ @@ -5115,6 +5344,219 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRoaringRequestView: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRoaringRequestView: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRoaringRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRoaringRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Clear", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Clear = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Views", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Views = append(m.Views, &ImportRoaringRequestView{}) + if err := m.Views[len(m.Views)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -5220,59 +5662,63 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_fc5da89825239896) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_5eafd62083455670) } -var fileDescriptor_public_fc5da89825239896 = []byte{ - // 804 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46, - 0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28, - 0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71, - 0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42, - 0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e, - 0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57, - 0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13, - 0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4, - 0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7, - 0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3, - 0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c, - 0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8, - 0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20, - 0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0, - 0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31, - 0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99, - 0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94, - 0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c, - 0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d, - 0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc, - 0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3, - 0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac, - 0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28, - 0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23, - 0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47, - 0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32, - 0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6, - 0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e, - 0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d, - 0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8, - 0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f, - 0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9, - 0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4, - 0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1, - 0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf, - 0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52, - 0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28, - 0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a, - 0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c, - 0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c, - 0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99, - 0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a, - 0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6, - 0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0, - 0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15, - 0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9, - 0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47, - 0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb, - 0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa, - 0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25, - 0x96, 0x07, 0x00, 0x00, +var fileDescriptor_public_5eafd62083455670 = []byte{ + // 870 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x8e, 0xdb, 0x44, + 0x14, 0x66, 0x62, 0x27, 0x71, 0x4e, 0x36, 0xa1, 0x1a, 0x2d, 0xc5, 0x42, 0x55, 0xb0, 0x2c, 0x84, + 0x7c, 0xb5, 0x95, 0x82, 0x54, 0xf5, 0x8a, 0x9f, 0x6d, 0xb6, 0x28, 0x2a, 0xac, 0xe0, 0x6c, 0x09, + 0xe2, 0xd2, 0x6d, 0xa6, 0xad, 0x25, 0xc7, 0x63, 0xec, 0x31, 0x69, 0x9e, 0x83, 0x1b, 0x1e, 0x81, + 0x0b, 0x1e, 0xa4, 0x97, 0x88, 0x27, 0x80, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0x64, 0x9c, 0x6c, 0x59, + 0x71, 0xc1, 0xdd, 0x7c, 0xe7, 0xcc, 0x39, 0xfe, 0xbe, 0x39, 0x3f, 0x09, 0x9c, 0x94, 0xcd, 0xb3, + 0x3c, 0x7b, 0x7e, 0x56, 0x56, 0x52, 0x49, 0x1e, 0x64, 0x85, 0x12, 0x55, 0x91, 0xe6, 0xf1, 0x0f, + 0xe0, 0xa1, 0xdc, 0xf2, 0x10, 0x86, 0x8f, 0x64, 0xde, 0x6c, 0x8a, 0x3a, 0x64, 0x91, 0x97, 0xf8, + 0x68, 0x21, 0xff, 0x08, 0xfa, 0x5f, 0x28, 0x55, 0xd5, 0x61, 0x2f, 0xf2, 0x92, 0xf1, 0x7c, 0x7a, + 0x66, 0x43, 0xcf, 0xb4, 0x19, 0x8d, 0x93, 0x73, 0xf0, 0x9f, 0x88, 0x5d, 0x1d, 0x7a, 0x91, 0x97, + 0x8c, 0x90, 0xce, 0xf1, 0x43, 0x98, 0xa2, 0xdc, 0x2e, 0xd7, 0xa2, 0x50, 0xd9, 0x8b, 0x4c, 0x98, + 0x5b, 0x28, 0xb7, 0xf6, 0x13, 0x74, 0xde, 0x47, 0xf6, 0x3a, 0x91, 0x9f, 0x82, 0xff, 0x4d, 0x9a, + 0x55, 0x7c, 0x0a, 0xbd, 0xe5, 0x22, 0x64, 0x11, 0x4b, 0x7c, 0xec, 0x2d, 0x17, 0xfc, 0x14, 0xfa, + 0x8f, 0x64, 0x53, 0xa8, 0xb0, 0x47, 0x26, 0x03, 0xf8, 0x1d, 0xf0, 0x9e, 0x88, 0x5d, 0xe8, 0x45, + 0x2c, 0x19, 0xa1, 0x3e, 0xc6, 0x0f, 0x20, 0x78, 0x9c, 0x89, 0x7c, 0xad, 0x95, 0x9d, 0x42, 0x9f, + 0xce, 0x94, 0x66, 0x84, 0x06, 0x68, 0xab, 0xe6, 0xb6, 0xb0, 0x99, 0x08, 0xc4, 0x5f, 0x01, 0x7c, + 0x59, 0xc9, 0xa6, 0x34, 0x79, 0x13, 0xe8, 0x13, 0x22, 0xba, 0xe3, 0x39, 0x77, 0xca, 0x6d, 0x72, + 0x34, 0x17, 0xde, 0xce, 0x2b, 0x9e, 0x43, 0xb0, 0x4a, 0xf3, 0x3d, 0xc7, 0x55, 0x9a, 0x13, 0x07, + 0x0f, 0xf5, 0xf1, 0x30, 0xc6, 0xb3, 0x31, 0xdf, 0x81, 0x77, 0x9e, 0x29, 0x47, 0x8f, 0x75, 0xe8, + 0xf1, 0x0f, 0x20, 0x30, 0x55, 0xd9, 0xf3, 0xde, 0x63, 0x7e, 0x0f, 0x46, 0x4f, 0xb3, 0x8d, 0xa8, + 0x55, 0xba, 0x29, 0xe9, 0x29, 0x3c, 0x74, 0x86, 0xf8, 0x7b, 0x98, 0x98, 0x9b, 0xba, 0x5a, 0x57, + 0x42, 0xdd, 0x78, 0xd9, 0xff, 0x56, 0xe5, 0x9b, 0x2f, 0xfd, 0x2b, 0x03, 0x5f, 0xfb, 0xac, 0x8b, + 0xed, 0x5d, 0xba, 0xb0, 0x4f, 0x77, 0xa5, 0x68, 0x99, 0xd2, 0x99, 0x47, 0x30, 0xbe, 0x52, 0x55, + 0x56, 0xbc, 0x5c, 0xa5, 0x79, 0x23, 0xda, 0x44, 0x5d, 0x93, 0xd6, 0xb8, 0x2c, 0x94, 0x71, 0xfb, + 0x24, 0x63, 0x8f, 0xb5, 0xc6, 0x73, 0x29, 0x73, 0xe3, 0xec, 0x47, 0x2c, 0x09, 0xd0, 0x19, 0xf8, + 0x0c, 0xe0, 0x71, 0x2e, 0xd3, 0x36, 0x76, 0x10, 0xb1, 0x84, 0x61, 0xc7, 0x12, 0xdf, 0x87, 0xa1, + 0x66, 0xfa, 0x75, 0x5a, 0x3a, 0xb5, 0xec, 0x16, 0xb5, 0xf1, 0x1b, 0x06, 0x27, 0xdf, 0x36, 0xa2, + 0xda, 0xa1, 0xf8, 0xb1, 0x11, 0x35, 0x55, 0x85, 0xb0, 0x6d, 0x25, 0x02, 0xfc, 0x2e, 0x0c, 0xae, + 0x5e, 0xa5, 0xd5, 0xda, 0xbc, 0x9d, 0x8f, 0x2d, 0xd2, 0x5a, 0xdd, 0x9b, 0xd7, 0xa4, 0x35, 0xc0, + 0xae, 0x49, 0x47, 0xa2, 0xd8, 0x48, 0x65, 0xc5, 0xb4, 0x88, 0x27, 0xf0, 0xee, 0xc5, 0xeb, 0xe7, + 0x79, 0xb3, 0x16, 0x28, 0xb7, 0x26, 0x7a, 0x40, 0x17, 0x8e, 0xcd, 0xfc, 0x63, 0x98, 0xb6, 0x26, + 0x3b, 0xbd, 0x43, 0xba, 0x78, 0x64, 0x8d, 0x7f, 0x66, 0x30, 0x69, 0xa5, 0xd4, 0xa5, 0x2c, 0x6a, + 0xa1, 0xeb, 0x75, 0x51, 0x55, 0xb6, 0x5e, 0x17, 0x55, 0xc5, 0xef, 0xc3, 0x10, 0x45, 0xdd, 0xe4, + 0xca, 0x36, 0xc1, 0x7b, 0xee, 0x59, 0x6c, 0x6c, 0x93, 0x2b, 0xb4, 0xb7, 0xf8, 0x67, 0x30, 0x3d, + 0x68, 0x2a, 0x33, 0xfd, 0xe3, 0xf9, 0xfb, 0x2e, 0xee, 0xc0, 0x8f, 0x47, 0xd7, 0xe3, 0x3f, 0x7a, + 0x30, 0xee, 0x64, 0xe6, 0x1f, 0xd2, 0x2e, 0x22, 0x4e, 0xe3, 0xf9, 0xc4, 0x65, 0xd1, 0x93, 0x46, + 0x5b, 0xea, 0x04, 0xd8, 0x65, 0xdb, 0x4f, 0xec, 0x52, 0x57, 0x51, 0x6f, 0x09, 0xfb, 0xd9, 0x4e, + 0x15, 0xb5, 0x19, 0x8d, 0x93, 0x36, 0xdb, 0xab, 0xb4, 0x78, 0x29, 0xd6, 0xd4, 0x4f, 0x01, 0x5a, + 0xc8, 0xcf, 0xdc, 0x7c, 0x52, 0x01, 0x0e, 0x46, 0xdc, 0x7a, 0xd0, 0xcd, 0xb0, 0x6d, 0x68, 0x5d, + 0x8b, 0x49, 0xdb, 0xd0, 0xba, 0x84, 0x7a, 0x36, 0xf5, 0xc3, 0x53, 0xf1, 0x0d, 0xe2, 0x0f, 0x60, + 0xec, 0x36, 0x49, 0x1d, 0x06, 0xc4, 0xf0, 0xd4, 0xa5, 0x77, 0x4e, 0xec, 0x5e, 0xe4, 0x9f, 0x1f, + 0xef, 0xcc, 0x70, 0x44, 0xcc, 0xc2, 0x83, 0xd7, 0xe8, 0xf8, 0xf1, 0xe8, 0x7e, 0xfc, 0x17, 0x83, + 0xc9, 0x72, 0x53, 0xca, 0x4a, 0x75, 0xda, 0x76, 0x59, 0xac, 0xc5, 0x6b, 0xdb, 0xb6, 0x04, 0xdc, + 0x5e, 0xec, 0x1d, 0xed, 0x45, 0x6a, 0x5f, 0x6a, 0x57, 0x1f, 0x0d, 0xe8, 0xa8, 0xf4, 0x0f, 0x54, + 0xde, 0x83, 0x91, 0x5d, 0x40, 0x75, 0xd8, 0x27, 0x97, 0x33, 0xe8, 0x81, 0xdc, 0x6f, 0x20, 0xdd, + 0xc1, 0x5e, 0xe2, 0x61, 0xc7, 0xa2, 0x2b, 0x83, 0x72, 0x4b, 0xcb, 0x7f, 0x48, 0xcb, 0xdf, 0x42, + 0x1d, 0x69, 0xd2, 0x90, 0x33, 0x20, 0x67, 0xc7, 0x12, 0xff, 0xc6, 0x80, 0x1b, 0x8d, 0x34, 0xda, + 0xff, 0x9f, 0xd0, 0xdb, 0x05, 0xdd, 0x85, 0x01, 0x7d, 0xcf, 0x8a, 0x69, 0xd1, 0x11, 0xdd, 0xe1, + 0x0d, 0xba, 0xe7, 0x10, 0xb6, 0x15, 0x91, 0xa9, 0xde, 0x74, 0x2d, 0xdf, 0x55, 0x26, 0xb6, 0xba, + 0xa9, 0x2e, 0xd3, 0x8d, 0x68, 0x29, 0xd3, 0x59, 0xdb, 0x16, 0xa9, 0x4a, 0x89, 0xf0, 0x09, 0xd2, + 0x39, 0x7e, 0x01, 0xa7, 0x6f, 0xcb, 0x41, 0x3f, 0x23, 0xb9, 0x48, 0xcd, 0x24, 0x07, 0x68, 0x00, + 0x7f, 0x08, 0xfd, 0x9f, 0x32, 0xb1, 0xb5, 0x93, 0x1c, 0xbb, 0xee, 0xf9, 0x37, 0x22, 0x68, 0x02, + 0xce, 0xef, 0xbc, 0xb9, 0x9e, 0xb1, 0xdf, 0xaf, 0x67, 0xec, 0xcf, 0xeb, 0x19, 0xfb, 0xe5, 0xef, + 0xd9, 0x3b, 0xcf, 0x06, 0xf4, 0x97, 0xe1, 0x93, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, 0x60, + 0x42, 0x9e, 0x42, 0x08, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 229fcfadd..67cef2a40 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -105,3 +105,13 @@ message ImportValueRequest { repeated string ColumnKeys = 7; repeated int64 Values = 6; } + +message ImportRoaringRequestView { + string Name = 1; + bytes Data = 2; +} + +message ImportRoaringRequest { + bool Clear = 1; + repeated ImportRoaringRequestView views = 2; +} \ No newline at end of file diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 96dae1433..e1bea3199 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -42,3 +42,25 @@ func TestValidateNameInvalid(t *testing.T) { } } } + +// memAttrStore represents an in-memory implementation of the AttrStore interface. +type memAttrStore struct { + store map[uint64]map[string]interface{} +} + +func (s *memAttrStore) Path() string { return "" } +func (s *memAttrStore) Open() error { return nil } +func (s *memAttrStore) Close() error { return nil } +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } +func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + s.store[id] = m + return nil +} +func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + for id, v := range m { + s.store[id] = v + } + return nil +} +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } diff --git a/server.go b/server.go index 24385c4ea..2e1e7091f 100644 --- a/server.go +++ b/server.go @@ -418,6 +418,9 @@ func (s *Server) loadNodeID() string { return nodeID } +// NodeID returns the server's node id. +func (s *Server) NodeID() string { return s.nodeID } + // SyncData manually invokes the anti entropy process which makes sure that this // node has the data from all replicas across the cluster. func (s *Server) SyncData() error { @@ -585,7 +588,6 @@ func (s *Server) SendSync(m Message) error { msg = append([]byte{getMessageType(m)}, msg...) for _, node := range s.cluster.nodes { node := node - s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.uri == node.URI { continue @@ -606,7 +608,6 @@ func (s *Server) SendAsync(m Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, m Message) error { - s.logger.Printf("SendTo: %s", to.URI) msg, err := s.serializer.Marshal(m) if err != nil { return fmt.Errorf("marshaling message: %v", err) @@ -630,7 +631,7 @@ func (s *Server) handleRemoteStatus(pb Message) { go func() { // Make sure the holder has opened. - <-s.holder.opened + s.holder.opened.Recv() err := s.mergeRemoteStatus(pb.(*NodeStatus)) if err != nil { @@ -658,7 +659,7 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { // if we don't know about a field locally, log an error because // fields should be created and synced prior to shard creation if f == nil { - s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name) + s.logger.Printf("local field not found: %s/%s", is.Name, fs.Name) continue } if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { @@ -703,7 +704,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.CheckVersion() err = s.diagnostics.Flush() if err != nil { - s.logger.Printf("Diagnostics error: %s", err) + s.logger.Printf("diagnostics error: %s", err) } } diff --git a/server/config.go b/server/config.go index d255e4bb6..6c159721e 100644 --- a/server/config.go +++ b/server/config.go @@ -19,6 +19,7 @@ import ( "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/toml" + "github.com/uber/jaeger-client-go" ) // TLSConfig contains TLS configuration @@ -91,6 +92,16 @@ type Config struct { // Pilosa's developers. Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` + + Tracing struct { + // SamplerType is the type of sampler to use. + SamplerType string `toml:"sampler-type"` + // SamplerParam is the parameter passed to the tracing sampler. + // Its meaning is dependent on the type of sampler. + SamplerParam float64 `toml:"sampler-param"` + // AgentHostPort is the host:port of the local agent. + AgentHostPort string `toml:"agent-host-port"` + } `toml:"tracing"` } // NewConfig returns an instance of Config with default options. @@ -133,5 +144,9 @@ func NewConfig() *Config { c.Metric.PollInterval = toml.Duration(0 * time.Minute) c.Metric.Diagnostics = true + // Tracing config. + c.Tracing.SamplerType = jaeger.SamplerTypeRemote + c.Tracing.SamplerParam = 0.001 + return c } diff --git a/server/handler_test.go b/server/handler_test.go index 9e76d77b9..5ae4dd877 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -22,15 +22,15 @@ import ( "fmt" "io" "io/ioutil" + gohttp "net/http" "net/http/httptest" "reflect" "strings" "testing" "time" - gohttp "net/http" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -91,9 +91,21 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - req := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(roaringData)) - req.Header.Set("Content-Type", "application/x-binary") - h.ServeHTTP(w, req) + msg := pilosa.ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, httpReq) resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) @@ -111,9 +123,21 @@ func TestHandler_Endpoints(t *testing.T) { } w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData)) - req.Header.Set("Content-Type", "application/x-binary") - h.ServeHTTP(w, req) + msg := pilosa.ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, httpReq) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } diff --git a/server/server.go b/server/server.go index 1e140d1f0..c6b69a7f5 100644 --- a/server/server.go +++ b/server/server.go @@ -142,7 +142,7 @@ func (m *Command) Start() (err error) { go func() { err := m.Handler.Serve() if err != nil { - m.logger.Printf("Handler serve error: %v", err) + m.logger.Printf("handler serve error: %v", err) } }() @@ -151,7 +151,7 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "opening server") } - m.logger.Printf("Listening as %s\n", m.API.Node().URI) + m.logger.Printf("listening as %s\n", m.API.Node().URI) return nil } @@ -163,27 +163,31 @@ func (m *Command) Wait() error { signal.Notify(c, os.Interrupt, syscall.SIGTERM) select { case sig := <-c: - m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) + m.logger.Printf("received signal '%s', gracefully shutting down...\n", sig.String()) // Second signal causes a hard shutdown. go func() { <-c; os.Exit(1) }() return errors.Wrap(m.Close(), "closing command") case <-m.done: - m.logger.Printf("Server closed externally") + m.logger.Printf("server closed externally") return nil } } // setupLogger sets up the logger based on the configuration. func (m *Command) setupLogger() error { - var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { - m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { return errors.Wrap(err, "opening file") } + m.logOutput = f + err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) + if err != nil { + return errors.Wrap(err, "dup2ing stderr onto logfile") + } } if m.Config.Verbose { @@ -372,6 +376,7 @@ func (m *Command) Close() error { eg.Go(closer.Close) } } + err := eg.Wait() return errors.Wrap(err, "closing everything") } diff --git a/server/server_test.go b/server/server_test.go index aa202a54c..3aa11b8a2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -621,6 +621,58 @@ func TestMain_ImportTimestamp(t *testing.T) { } } +func TestMain_ImportTimestampNoStandardView(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + indexName := "i" + fieldName := "f-no-standard" + + // Create index. + if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + // Create field. + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)); err != nil { + t.Fatal(err) + } + + data := pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + RowIDs: []uint64{1, 2}, + ColumnIDs: []uint64{1, 2}, + Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00 + } + + // Import data. + if err := m.API.Import(context.Background(), &data); err != nil { + t.Fatal(err) + } + + // Ensure the correct views were created. + dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName) + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + exp := []string{ + "standard_2018", "standard_201801", "standard_20180101", + "standard_2019", "standard_201912", "standard_20191231", + } + got := []string{} + for _, f := range files { + got = append(got, f.Name()) + } + + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected %v, but got %v", exp, got) + } +} + func TestClusterQueriesAfterRestart(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go new file mode 100644 index 000000000..aacc42d6e --- /dev/null +++ b/tracing/opentracing/opentracing.go @@ -0,0 +1,60 @@ +package opentracing + +import ( + "context" + "log" + "net/http" + + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" + "github.com/pilosa/pilosa/tracing" +) + +// Ensure type implements interface. +var _ tracing.Tracer = (*Tracer)(nil) + +// Tracer represents a wrapper for OpenTracing that implements tracing.Tracer. +type Tracer struct { + tracer opentracing.Tracer +} + +// NewTracer returns a new instance of Tracer. +func NewTracer(tracer opentracing.Tracer) *Tracer { + return &Tracer{tracer: tracer} +} + +// StartSpanFromContext returns a new child span and context from a given context. +func (t *Tracer) StartSpanFromContext(ctx context.Context, operationName string) (tracing.Span, context.Context) { + var opts []opentracing.StartSpanOption + if parent := opentracing.SpanFromContext(ctx); parent != nil { + opts = append(opts, opentracing.ChildOf(parent.Context())) + } + span := t.tracer.StartSpan(operationName, opts...) + return span, opentracing.ContextWithSpan(ctx, span) +} + +// InjectHTTPHeaders adds the required HTTP headers to pass context between nodes. +func (t *Tracer) InjectHTTPHeaders(r *http.Request) { + if span := opentracing.SpanFromContext(r.Context()); span != nil { + if err := t.tracer.Inject( + span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(r.Header), + ); err != nil { + log.Printf("opentracing inject error: %s", err) + } + } +} + +// ExtractHTTPHeaders reads the HTTP headers to derive incoming context. +func (t *Tracer) ExtractHTTPHeaders(r *http.Request) (tracing.Span, context.Context) { + // Deserialize tracing context into request. + wireContext, _ := t.tracer.Extract( + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(r.Header), + ) + + span := t.tracer.StartSpan("HTTP", ext.RPCServerOption(wireContext)) + ctx := opentracing.ContextWithSpan(r.Context(), span) + return span, ctx +} diff --git a/tracing/tracing.go b/tracing/tracing.go new file mode 100644 index 000000000..5792dd7ca --- /dev/null +++ b/tracing/tracing.go @@ -0,0 +1,58 @@ +package tracing + +import ( + "context" + "net/http" +) + +// GlobalTracer is a single, global instance of Tracer. +var GlobalTracer Tracer = NopTracer() + +// StartSpanFromContext returnus a new child span and context from a given +// context using the global tracer. +func StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context) { + return GlobalTracer.StartSpanFromContext(ctx, operationName) +} + +// Tracer implements a generic distributed tracing interface. +type Tracer interface { + // Returns a new child span and context from a given context. + StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context) + + // Adds the required HTTP headers to pass context between nodes. + InjectHTTPHeaders(r *http.Request) + + // Reads the HTTP headers to derive incoming context. + ExtractHTTPHeaders(r *http.Request) (Span, context.Context) +} + +// Span represents a single span in a distributed trace. +type Span interface { + // Sets the end timestamp and finalizes Span state. + Finish() + + // Adds key/value pairs to the span. + LogKV(alternatingKeyValues ...interface{}) +} + +// NopTracer returns a tracer that doesn't do anything. +func NopTracer() Tracer { + return &nopTracer{} +} + +type nopTracer struct{} + +func (t *nopTracer) StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context) { + return &nopSpan{}, ctx +} + +func (t *nopTracer) InjectHTTPHeaders(r *http.Request) {} + +func (t *nopTracer) ExtractHTTPHeaders(r *http.Request) (Span, context.Context) { + return &nopSpan{}, r.Context() +} + +type nopSpan struct{} + +func (s *nopSpan) Finish() {} +func (s *nopSpan) LogKV(alternatingKeyValues ...interface{}) {} diff --git a/translate.go b/translate.go index 669e1e323..2c3125ad9 100644 --- a/translate.go +++ b/translate.go @@ -195,12 +195,11 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error { } // Stop translate store replication. - s.logger.Printf("stop monitor replication") close(s.replicationClosing) s.repWG.Wait() // Set the primary node for translate store replication. - s.logger.Printf("set primary translate store to %s", ev.id) + s.logger.Debugf("set primary translate store to %s", ev.id) s.primaryID = ev.id if ev.id == "" { s.PrimaryTranslateStore = nil @@ -209,7 +208,6 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error { } // Start translate store replication. Stream from primary, if available. - s.logger.Printf("start monitor replication") if s.PrimaryTranslateStore != nil { s.replicationClosing = make(chan struct{}) s.repWG.Add(1) @@ -386,7 +384,6 @@ func (s *TranslateFile) monitorReplication() { // monitorPrimaryStoreEvents is executed in a separate goroutine and listens for changes // to the primary store assignment. func (s *TranslateFile) monitorPrimaryStoreEvents() { - s.logger.Printf("monitor primary store events") // Keep handling events until the store closes. for { select { @@ -404,7 +401,7 @@ func (s *TranslateFile) replicate(ctx context.Context) error { off := s.size() // Connect to remote primary. - s.logger.Printf("pilosa: replicating from offset %d", off) + s.logger.Debugf("pilosa: replicating from offset %d", off) rc, err := s.PrimaryTranslateStore.Reader(ctx, off) if err != nil { return err