scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'shift-op' of github.com:tgruben/pilosa into shift-op

Browse source
This commit is contained in:
Todd Gruben 2018-11-26 15:06:33 -06:00
commit ba0ad350a9
36 changed files with 1841 additions and 472 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
.circleci/config.yml
View file

@ -2,7 +2,7 @@ version: 2
defaults: &defaults
working_directory: /go/src/github.com/pilosa/pilosa
docker:
- image: circleci/golang:1.10
- image: circleci/golang:1.11
fast-checkout: &fast-checkout
attach_workspace:
at: .
@ -30,20 +30,28 @@ jobs:
- run: gometalinter --install
- run: go get github.com/remyoudompheng/go-misc/deadcode
- run: make gometalinter
test-golang-1.10: &base-test
test-golang-1.11: &base-test
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run: make test
test-golang-1.11-rc:
<<: *base-test
docker:
- image: circleci/golang:1.11-rc
test-golang-1.10-386:
test-golang-1.11-race:
<<: *defaults
steps:
- *fast-checkout
- run: sudo apt-get install lsof
- run:
command: make test TESTFLAGS="-race -timeout=30m"
no_output_timeout: 30m
test-golang-1.11-386:
<<: *base-test
environment:
GOARCH: 386
test-golang-1.10:
<<: *base-test
docker:
- image: circleci/golang:1.10
cluster-tests:
<<: *defaults
steps:
@ -96,26 +104,29 @@ workflows:
- linter:
requires:
- build
- test-golang-1.11:
requires:
- build
- test-golang-1.11-race:
requires:
- build
- test-golang-1.11-386:
requires:
- build
- test-golang-1.10:
requires:
- build
- test-golang-1.11-rc:
requires:
- build
- test-golang-1.10-386:
requires:
- build
- cluster-tests:
requires:
- build
- prerelease:
requires:
- linter
- test-golang-1.10
- test-golang-1.11
- release:
requires:
- linter
- test-golang-1.10
- test-golang-1.11
filters:
tags:
only: /^v.*/
@ -127,4 +138,4 @@ workflows:
- dockerhub-upload:
requires:
- linter
- test-golang-1.10
- test-golang-1.11

205
Gopkg.lock generated
View file

@ -3,141 +3,192 @@
[[projects]]
branch = "master"
digest = "1:7deffcad61694d0cf57178c7d0c41df6b1357612e215914e35cf8b6353abf65c"
name = "github.com/CAFxX/gcnotifier"
packages = ["."]
pruneopts = ""
revision = "39b0596a2da3c92787b3319c6b5425a474b4e0da"
[[projects]]
branch = "master"
digest = "1:7519fc1e7fa9cde38634cfaf8a2ba842ea20aab2c14d81d6122ab8d6b1658d1c"
name = "github.com/DataDog/datadog-go"
packages = ["statsd"]
pruneopts = ""
revision = "ef3a9daf849df2d7ee3bbf13808dfb481069a773"
[[projects]]
digest = "1:f82b8ac36058904227087141017bb82f4b0fc58272990a4cdae3e2d6d222644e"
name = "github.com/StackExchange/wmi"
packages = ["."]
pruneopts = ""
revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338"
version = "1.0.0"
[[projects]]
branch = "master"
digest = "1:354e62d5acb9af138e13ec842f78a846d214a8d4a9f80e578698f1f1565e2ef8"
name = "github.com/armon/go-metrics"
packages = ["."]
pruneopts = ""
revision = "3c58d8115a78a6879e5df75ae900846768d36895"
[[projects]]
digest = "1:ed112122ed4a920d944cc99b9d00b0441c11685939c28462c719488d36fe29aa"
name = "github.com/boltdb/bolt"
packages = ["."]
pruneopts = ""
revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
version = "v1.3.1"
[[projects]]
digest = "1:1660bb2e30cca08494f29b5593e387c6090fbe8936970ba947185b0ca000aec0"
name = "github.com/cespare/xxhash"
packages = ["."]
pruneopts = ""
revision = "5c37fe3735342a2e0d01c87a907579987c8936cc"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:c46fd324e7902268373e1b337436a6377c196e2dbd7b35624c6256d29d494e78"
name = "github.com/codahale/hdrhistogram"
packages = ["."]
pruneopts = ""
revision = "3a0bb77429bd3a61596f5e8a3172445844342120"
[[projects]]
digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
name = "github.com/fsnotify/fsnotify"
packages = ["."]
pruneopts = ""
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.7"
[[projects]]
digest = "1:96c4a6ff4206086347bfe28e96e092642882128f45ecb8dc8f15f3e6f6703af0"
name = "github.com/go-ole/go-ole"
packages = [
".",
"oleutil"
"oleutil",
]
pruneopts = ""
revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506"
version = "v1.2.1"
[[projects]]
digest = "1:6e73003ecd35f4487a5e88270d3ca0a81bc80dc88053ac7e4dcfec5fba30d918"
name = "github.com/gogo/protobuf"
packages = ["proto"]
pruneopts = ""
revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
version = "v1.1.1"
[[projects]]
digest = "1:f958a1c137db276e52f0b50efee41a1a389dcdded59a69711f3e872757dab34b"
name = "github.com/golang/protobuf"
packages = ["proto"]
pruneopts = ""
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"
[[projects]]
digest = "1:f9f45f75f332e03fc7e9fe9188ea4e1ce4d14779ef34fa1b023da67518e36327"
name = "github.com/google/go-cmp"
packages = [
"cmp",
"cmp/cmpopts",
"cmp/internal/diff",
"cmp/internal/function",
"cmp/internal/value"
"cmp/internal/value",
]
pruneopts = ""
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
version = "v0.2.0"
[[projects]]
digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
name = "github.com/gorilla/context"
packages = ["."]
pruneopts = ""
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
version = "v1.1.1"
[[projects]]
digest = "1:a1a1522a8c1fad5675ce8ec9de96f30b898df7c8c229928c35dbd8cf277e6d62"
name = "github.com/gorilla/handlers"
packages = ["."]
pruneopts = ""
revision = "90663712d74cb411cbef281bc1e08c19d1a76145"
version = "v1.3.0"
[[projects]]
digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
name = "github.com/gorilla/mux"
packages = ["."]
pruneopts = ""
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
version = "v1.6.2"
[[projects]]
branch = "master"
digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
name = "github.com/hashicorp/errwrap"
packages = ["."]
pruneopts = ""
revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
[[projects]]
branch = "master"
digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
name = "github.com/hashicorp/go-immutable-radix"
packages = ["."]
pruneopts = ""
revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
[[projects]]
branch = "master"
digest = "1:6396690228a7560bf9247cb90e5ae9c797bd630b01e7d2acab430bbca9a1ecb3"
name = "github.com/hashicorp/go-msgpack"
packages = ["codec"]
pruneopts = ""
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
[[projects]]
branch = "master"
digest = "1:0b5ca7d18e4ded1e4dacbb37ff027cb40a80c0fed969e4e03cf7aff129bc1b44"
name = "github.com/hashicorp/go-multierror"
packages = ["."]
pruneopts = ""
revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
[[projects]]
branch = "master"
digest = "1:fd8ec2359315965bb6b84fd8e45cd5e8b58b80d8430dc96c8c5dfce46d30dbfc"
name = "github.com/hashicorp/go-sockaddr"
packages = ["."]
pruneopts = ""
revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
[[projects]]
branch = "master"
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
name = "github.com/hashicorp/golang-lru"
packages = ["simplelru"]
pruneopts = ""
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
[[projects]]
branch = "master"
digest = "1:9b7c5846d70f425d7fe279595e32a20994c6075e87be03b5c367ed07280877c5"
name = "github.com/hashicorp/hcl"
packages = [
".",
@ -149,142 +200,207 @@
"hcl/token",
"json/parser",
"json/scanner",
"json/token"
"json/token",
]
pruneopts = ""
revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168"
[[projects]]
digest = "1:d2c45a353b65012162c7ca22c39b1b0bd06d39362fb375cf42b4e48e1104bfc6"
name = "github.com/hashicorp/memberlist"
packages = ["."]
pruneopts = ""
revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
version = "v0.1.0"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
pruneopts = ""
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
name = "github.com/magiconair/properties"
packages = ["."]
pruneopts = ""
revision = "c2353362d570a7bfa228149c62842019201cfb71"
version = "v1.8.0"
[[projects]]
digest = "1:4c8d8358c45ba11ab7bb15df749d4df8664ff1582daead28bae58cf8cbe49890"
name = "github.com/miekg/dns"
packages = ["."]
pruneopts = ""
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
version = "v1.0.8"
[[projects]]
branch = "master"
digest = "1:f43ed2c836208c14f45158fd01577c985688a4d11cf9fd475a939819fef3b321"
name = "github.com/mitchellh/mapstructure"
packages = ["."]
pruneopts = ""
revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac"
[[projects]]
digest = "1:78fb99d6011c2ae6c72f3293a83951311147b12b06a5ffa43abf750c4fab6ac5"
name = "github.com/opentracing/opentracing-go"
packages = [
".",
"ext",
"log",
]
pruneopts = ""
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
name = "github.com/pelletier/go-toml"
packages = ["."]
pruneopts = ""
revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
version = "v1.2.0"
[[projects]]
name = "github.com/pilosa/go-pilosa"
packages = [
".",
"gopilosa_pbuf"
]
revision = "4e7807f5ad779407936744057cd17332046b6c3c"
version = "v1.1.0"
[[projects]]
digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = ""
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
digest = "1:7f569d906bdd20d906b606415b7d794f798f91a62fcfb6a4daa6d50690fb7a3f"
name = "github.com/satori/go.uuid"
packages = ["."]
pruneopts = ""
revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
name = "github.com/sean-/seed"
packages = ["."]
pruneopts = ""
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
[[projects]]
digest = "1:02715a2fb4b9279af36651a59a51dd4164eb689bd6785874811899f43eeb2a54"
name = "github.com/shirou/gopsutil"
packages = [
"cpu",
"host",
"internal/common",
"mem",
"net",
"process"
"process",
]
pruneopts = ""
revision = "8048a2e9c5773235122027dd585cf821b2af1249"
version = "v2.18.07"
[[projects]]
branch = "master"
name = "github.com/shirou/w32"
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
digest = "1:7ba2551c9a8de293bc575dbe2c0d862c52252d26f267f784547f059f512471c8"
name = "github.com/spf13/afero"
packages = [
".",
"mem"
"mem",
]
pruneopts = ""
revision = "787d034dfe70e44075ccc060d346146ef53270ad"
version = "v1.1.1"
[[projects]]
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
name = "github.com/spf13/cast"
packages = ["."]
pruneopts = ""
revision = "8965335b8c7107321228e3e3702cab9832751bac"
version = "v1.2.0"
[[projects]]
digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
name = "github.com/spf13/cobra"
packages = ["."]
pruneopts = ""
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
version = "v0.0.3"
[[projects]]
branch = "master"
digest = "1:104517520aab91164020ab6524a5d6b7cafc641b2e42ac6236f6ac1deac4f66a"
name = "github.com/spf13/jwalterweatherman"
packages = ["."]
pruneopts = ""
revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
[[projects]]
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = ""
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
version = "v1.0.1"
[[projects]]
digest = "1:3dab237cd3263a290d771d133fed777bb56c22e380b00ebe92e6531d5c8d3d0c"
name = "github.com/spf13/viper"
packages = ["."]
pruneopts = ""
revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736"
version = "v1.0.2"
[[projects]]
digest = "1:941ab4973b3218a9a6d02d31734f5c762239eb538c0d702fff62d4037af8ab0a"
name = "github.com/uber/jaeger-client-go"
packages = [
".",
"config",
"internal/baggage",
"internal/baggage/remote",
"internal/spanlog",
"internal/throttler",
"internal/throttler/remote",
"log",
"rpcmetrics",
"thrift",
"thrift-gen/agent",
"thrift-gen/baggage",
"thrift-gen/jaeger",
"thrift-gen/sampling",
"thrift-gen/zipkincore",
"transport",
"utils",
]
pruneopts = ""
revision = "1a782e2da844727691fef1757c72eb190c2909f0"
version = "v2.15.0"
[[projects]]
digest = "1:aa1598d34009b45ce74fdabdd25e4258d7923d1e1b418d4c98482e79607cb9b0"
name = "github.com/uber/jaeger-lib"
packages = ["metrics"]
pruneopts = ""
revision = "ed3a127ec5fef7ae9ea95b01b542c47fbd999ce5"
version = "v1.5.0"
[[projects]]
branch = "master"
digest = "1:cae234a803b78380e4d769db6036b9fcc8c08ed4ff862571ffc1a958edc1f629"
name = "golang.org/x/crypto"
packages = [
"ed25519",
"ed25519/internal/edwards25519"
"ed25519/internal/edwards25519",
]
pruneopts = ""
revision = "c126467f60eb25f8f27e5a981f32a87e3965053f"
[[projects]]
branch = "master"
digest = "1:1f3b488bf9c50cac0cb738fca05d8ff55509c8698f58ea4d74b1499d0baaeb74"
name = "golang.org/x/net"
packages = [
"bpf",
@ -292,26 +408,32 @@
"internal/iana",
"internal/socket",
"ipv4",
"ipv6"
"ipv6",
]
pruneopts = ""
revision = "22bb95c5e783d192c577a7b310b06637db9f1d94"
[[projects]]
branch = "master"
digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
name = "golang.org/x/sync"
packages = ["errgroup"]
pruneopts = ""
revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
[[projects]]
branch = "master"
digest = "1:dd631ee90bd2e7aa16b6e094217d77a797684b52811374c948c695cbb46b5bbb"
name = "golang.org/x/sys"
packages = [
"unix",
"windows"
"windows",
]
pruneopts = ""
revision = "bd9dbc187b6e1dacfdd2722a87e83093c2d7bd6e"
[[projects]]
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text"
packages = [
"internal/gen",
@ -319,20 +441,49 @@
"internal/ucd",
"transform",
"unicode/cldr",
"unicode/norm"
"unicode/norm",
]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = ""
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "be318fa4f2a72e7e849b2faff1ce9300deaaf2d24cf76100f4b538abed86295b"
input-imports = [
"github.com/CAFxX/gcnotifier",
"github.com/DataDog/datadog-go/statsd",
"github.com/boltdb/bolt",
"github.com/cespare/xxhash",
"github.com/davecgh/go-spew/spew",
"github.com/gogo/protobuf/proto",
"github.com/golang/protobuf/proto",
"github.com/google/go-cmp/cmp",
"github.com/google/go-cmp/cmp/cmpopts",
"github.com/gorilla/handlers",
"github.com/gorilla/mux",
"github.com/hashicorp/memberlist",
"github.com/opentracing/opentracing-go",
"github.com/opentracing/opentracing-go/ext",
"github.com/pelletier/go-toml",
"github.com/pkg/errors",
"github.com/satori/go.uuid",
"github.com/shirou/gopsutil/host",
"github.com/shirou/gopsutil/mem",
"github.com/spf13/cobra",
"github.com/spf13/pflag",
"github.com/spf13/viper",
"github.com/uber/jaeger-client-go",
"github.com/uber/jaeger-client-go/config",
"golang.org/x/sync/errgroup",
]
solver-name = "gps-cdcl"
solver-version = 1

26
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-dep install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build require-dep require-gometalinter require-protoc require-protoc-gen-gofast require-peg test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql gometalinter install install-build-deps install-dep install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -150,26 +150,10 @@ gometalinter: require-gometalinter
######################
# Verifies that needed build dependency is installed. Errors out if not installed.
define require
$(if $(shell command -v $1 2>/dev/null),
$(info Verified build dependency "$1" is installed.),
$(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`))
endef
require-dep:
$(call require,dep)
require-protoc-gen-gofast:
$(call require,protoc-gen-gofast)
require-protoc:
$(call require,protoc)
require-peg:
$(call require,peg)
require-gometalinter:
$(call require,gometalinter)
require-%:
$(if $(shell command -v $* 2>/dev/null),\
$(info Verified build dependency "$*" is installed.),\
$(error Build dependency "$*" not installed. To install, try `make install-$*`))
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg

166
api.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/stats"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -99,6 +100,9 @@ func (api *API) validate(f apiMethod) error {
// Query parses a PQL query out of the request and executes it.
func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.Query")
defer span.Finish()
if err := api.validate(apiQuery); err != nil {
return QueryResponse{}, errors.Wrap(err, "validating api method")
}
@ -122,7 +126,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
}
// CreateIndex makes a new Pilosa index.
func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) {
func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.CreateIndex")
defer span.Finish()
if err := api.validate(apiCreateIndex); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -146,7 +153,10 @@ func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOp
}
// Index retrieves the named index.
func (api *API) Index(_ context.Context, indexName string) (*Index, error) {
func (api *API) Index(ctx context.Context, indexName string) (*Index, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Index")
defer span.Finish()
if err := api.validate(apiIndex); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -160,7 +170,10 @@ func (api *API) Index(_ context.Context, indexName string) (*Index, error) {
// DeleteIndex removes the named index. If the index is not found it does
// nothing and returns no error.
func (api *API) DeleteIndex(_ context.Context, indexName string) error {
func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteIndex")
defer span.Finish()
if err := api.validate(apiDeleteIndex); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -186,7 +199,10 @@ func (api *API) DeleteIndex(_ context.Context, indexName string) error {
// CreateField makes the named field in the named index with the given options.
// This method currently only takes a single functional option, but that may be
// changed in the future to support multiple options.
func (api *API) CreateField(_ context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
defer span.Finish()
if err := api.validate(apiCreateField); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -228,7 +244,10 @@ func (api *API) CreateField(_ context.Context, indexName string, fieldName strin
}
// Field retrieves the named field.
func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, error) {
func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Field")
defer span.Finish()
if err := api.validate(apiField); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -268,21 +287,14 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) {
// (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all
// of the rows in this shard of this field concatenated together in one long
// bitmap.
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, data []byte, opts ...ImportOption) (err error) {
if len(data) == 0 {
return errors.New("no data to import")
}
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring")
defer span.Finish()
if err = api.validate(apiField); err != nil {
return errors.Wrap(err, "validating api method")
}
// Set up import options.
options, err := setUpImportOptions(opts...)
if err != nil {
return errors.Wrap(err, "setting up import options")
}
nodes := api.cluster.shardNodes(indexName, shard)
var eg errgroup.Group
@ -291,26 +303,42 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return newNotFoundError(ErrFieldNotFound)
}
// only set fields are supported
if field.Type() != FieldTypeSet {
return NewBadRequestError(errors.New("roaring import is only supported for set fields"))
// only set and time fields are supported
if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime {
return NewBadRequestError(errors.New("roaring import is only supported for set and time fields"))
}
for _, node := range nodes {
node := node
if node.ID == api.server.nodeID {
// must make a copy of data to operate on locally. field.importRoaring changes data
d2 := make([]byte, len(data))
copy(d2, data)
eg.Go(func() error {
return field.importRoaring(d2, shard, options.Clear)
var err error
for viewName, viewData := range req.Views {
if viewName == "" {
viewName = viewStandard
} else {
viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
}
if len(viewData) == 0 {
return fmt.Errorf("no data to import for view: %s", viewName)
}
// must make a copy of data to operate on locally.
// field.importRoaring changes data
data := make([]byte, len(viewData))
copy(data, viewData)
err = field.importRoaring(data, shard, viewName, req.Clear)
if err != nil {
return err
}
}
return err
})
go func(node *Node) {
}(node)
} else if !remote { // if remote == true we don't forward to other nodes
// forward it on
eg.Go(func() error {
return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data, opts...)
return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, req)
})
}
}
@ -320,7 +348,10 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
// DeleteField removes the named field from the named index. If the index is not
// found, an error is returned. If the field is not found, it is ignored and no
// action is taken.
func (api *API) DeleteField(_ context.Context, indexName string, fieldName string) error {
func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteField")
defer span.Finish()
if err := api.validate(apiDeleteField); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -384,7 +415,10 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
// ExportCSV encodes the fragment designated by the index,field,shard as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error {
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.ExportCSV")
defer span.Finish()
if err := api.validate(apiExportCSV); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -418,6 +452,7 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string,
// Define the function to write each bit as a string,
// translating to keys where necessary.
var n int
fn := func(rowID, columnID uint64) error {
var rowStr string
var colStr string
@ -439,6 +474,7 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string,
colStr = strconv.FormatUint(columnID, 10)
}
n++
return cw.Write([]string{rowStr, colStr})
}
@ -450,11 +486,16 @@ func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string,
// Ensure data is flushed.
cw.Flush()
span.LogKV("n", n)
return nil
}
// ShardNodes returns the node and all replicas which should contain a shard's data.
func (api *API) ShardNodes(_ context.Context, indexName string, shard uint64) ([]*Node, error) {
func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes")
defer span.Finish()
if err := api.validate(apiShardNodes); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -465,7 +506,10 @@ func (api *API) ShardNodes(_ context.Context, indexName string, shard uint64) ([
// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to
// return anything useful. Currently it returns protobuf encoded row and column
// ids from a "block" which is a subdivision of a fragment.
func (api *API) FragmentBlockData(_ context.Context, body io.Reader) ([]byte, error) {
func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlockData")
defer span.Finish()
if err := api.validate(apiFragmentBlockData); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -498,7 +542,10 @@ func (api *API) FragmentBlockData(_ context.Context, body io.Reader) ([]byte, er
}
// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (api *API) FragmentBlocks(_ context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error) {
func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlocks")
defer span.Finish()
if err := api.validate(apiFragmentBlocks); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -516,7 +563,9 @@ func (api *API) FragmentBlocks(_ context.Context, indexName, fieldName, viewName
// Hosts returns a list of the hosts in the cluster including their ID,
// URL, and which is the coordinator.
func (api *API) Hosts(_ context.Context) []*Node {
func (api *API) Hosts(ctx context.Context) []*Node {
span, _ := tracing.StartSpanFromContext(ctx, "API.Hosts")
defer span.Finish()
return api.cluster.Nodes()
}
@ -527,7 +576,10 @@ func (api *API) Node() *Node {
}
// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
func (api *API) RecalculateCaches(_ context.Context) error {
func (api *API) RecalculateCaches(ctx context.Context) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.RecalculateCaches")
defer span.Finish()
if err := api.validate(apiRecalculateCaches); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -542,7 +594,10 @@ func (api *API) RecalculateCaches(_ context.Context) error {
// PostClusterMessage is for internal use. It decodes a protobuf message out of
// the body and forwards it to the BroadcastHandler.
func (api *API) ClusterMessage(_ context.Context, reqBody io.Reader) error {
func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.ClusterMessage")
defer span.Finish()
if err := api.validate(apiClusterMessage); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -569,12 +624,17 @@ func (api *API) ClusterMessage(_ context.Context, reqBody io.Reader) error {
// Schema returns information about each index in Pilosa including which fields
// they contain.
func (api *API) Schema(_ context.Context) []*IndexInfo {
func (api *API) Schema(ctx context.Context) []*IndexInfo {
span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
defer span.Finish()
return api.holder.limitedSchema()
}
// Views returns the views in the given field.
func (api *API) Views(_ context.Context, indexName string, fieldName string) ([]*view, error) {
func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Views")
defer span.Finish()
if err := api.validate(apiViews); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -591,7 +651,10 @@ func (api *API) Views(_ context.Context, indexName string, fieldName string) ([]
}
// DeleteView removes the given view.
func (api *API) DeleteView(_ context.Context, indexName string, fieldName string, viewName string) error {
func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.DeleteView")
defer span.Finish()
if err := api.validate(apiDeleteView); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -625,7 +688,10 @@ func (api *API) DeleteView(_ context.Context, indexName string, fieldName string
}
// IndexAttrDiff
func (api *API) IndexAttrDiff(_ context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff")
defer span.Finish()
if err := api.validate(apiIndexAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -659,7 +725,10 @@ func (api *API) IndexAttrDiff(_ context.Context, indexName string, blocks []Attr
return attrs, nil
}
func (api *API) FieldAttrDiff(_ context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff")
defer span.Finish()
if err := api.validate(apiFieldAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -718,6 +787,9 @@ func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption {
// Import bulk imports data into a particular index,field,shard.
func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
defer span.Finish()
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -823,6 +895,9 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
// ImportValue bulk imports values into a particular field.
func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue")
defer span.Finish()
if err := api.validate(apiImportValue); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -916,7 +991,10 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error {
// MaxShards returns the maximum shard number for each index in a map.
// TODO (2.0): This method has been deprecated. Instead, use
// AvailableShardsByIndex.
func (api *API) MaxShards(_ context.Context) map[string]uint64 {
func (api *API) MaxShards(ctx context.Context) map[string]uint64 {
span, _ := tracing.StartSpanFromContext(ctx, "API.MaxShards")
defer span.Finish()
m := make(map[string]uint64)
for k, v := range api.holder.availableShardsByIndex() {
m[k] = v.Max()
@ -925,7 +1003,9 @@ func (api *API) MaxShards(_ context.Context) map[string]uint64 {
}
// AvailableShardsByIndex returns bitmaps of shards with available by index name.
func (api *API) AvailableShardsByIndex(_ context.Context) map[string]*roaring.Bitmap {
func (api *API) AvailableShardsByIndex(ctx context.Context) map[string]*roaring.Bitmap {
span, _ := tracing.StartSpanFromContext(ctx, "API.AvailableShardsByIndex")
defer span.Finish()
return api.holder.availableShardsByIndex()
}
@ -957,7 +1037,7 @@ func (api *API) validateShardOwnership(indexName string, shard uint64) error {
}
func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) {
api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard)
api.server.logger.Debugf("importing: %v %v %v", indexName, fieldName, shard)
// Find the Index.
index := api.holder.Index(indexName)
@ -976,7 +1056,10 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I
}
// SetCoordinator makes a new Node the cluster coordinator.
func (api *API) SetCoordinator(_ context.Context, id string) (oldNode, newNode *Node, err error) {
func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.SetCoordinator")
defer span.Finish()
if err := api.validate(apiSetCoordinator); err != nil {
return nil, nil, errors.Wrap(err, "validating api method")
}
@ -1041,6 +1124,9 @@ func (api *API) ResizeAbort() error {
// GetTranslateData provides a reader for key translation logs starting at offset.
func (api *API) GetTranslateData(ctx context.Context, offset int64) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateData")
defer span.Finish()
rc, err := api.holder.translateFile.Reader(ctx, offset)
if err != nil {
return nil, errors.Wrap(err, "read from translate store")

22
attr.go
View file

@ -203,25 +203,3 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) {
}
return decodeAttrs(pb.GetAttrs()), nil
}
// memAttrStore represents an in-memory implementation of the AttrStore interface.
type memAttrStore struct {
store map[uint64]map[string]interface{}
}
func (s *memAttrStore) Path() string { return "" }
func (s *memAttrStore) Open() error { return nil }
func (s *memAttrStore) Close() error { return nil }
func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil }
func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
s.store[id] = m
return nil
}
func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
for id, v := range m {
s.store[id] = v
}
return nil
}
func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }

4
client.go
View file

@ -53,7 +53,7 @@ type InternalClient interface {
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error
ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
}
//===============
@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar
func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error {
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error {
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {

29
cluster.go
View file

@ -27,14 +27,14 @@ import (
"sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/logger"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"golang.org/x/sync/errgroup"
)
const (
@ -347,8 +347,6 @@ func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool {
// addNode adds a node to the Cluster and updates and saves the
// new topology. unprotected.
func (c *cluster) addNode(node *Node) error {
c.logger.Printf("add node %s to cluster on %s", node, c.Node)
// If the node being added is the coordinator, set it for this node.
if node.IsCoordinator {
c.Coordinator = node.ID
@ -481,7 +479,7 @@ func (c *cluster) setNodeState(state string) error { // nolint: unparam
State: state,
}
c.logger.Printf("Sending State %s (%s)", state, c.Coordinator)
c.logger.Printf("sending state %s (%s)", state, c.Coordinator)
if err := c.sendTo(c.coordinatorNode(), ns); err != nil {
return fmt.Errorf("sending node state error: err=%s", err)
}
@ -970,7 +968,6 @@ func (c *cluster) close() error {
}
func (c *cluster) markAsJoined() {
c.logger.Printf("mark node as joined (received coordinator update)")
if !c.joined {
c.joined = true
close(c.joining)
@ -1069,7 +1066,6 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
}
// Broadcast cluster status changes to the cluster.
status := c.unprotectedStatus()
c.logger.Printf("broadcasting ClusterStatus: %s", status)
return c.broadcaster.SendSync(status) // TODO fix c.Status
}
@ -1246,14 +1242,14 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
return errors.Wrap(err, "merging cluster status")
}
c.logger.Printf("MergeClusterStatus done, start goroutine")
c.logger.Printf("done MergeClusterStatus, start goroutine")
// The actual resizing runs in a goroutine because we don't want to block
// the distribution of other ResizeInstructions to the rest of the cluster.
go func() {
// Make sure the holder has opened.
<-c.holder.opened
c.holder.opened.Recv()
// Prepare the return message.
complete := &ResizeInstructionComplete{
@ -1264,9 +1260,11 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
// Stop processing on any error.
if err := func() error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "Cluster.followResizeInstruction")
defer span.Finish()
// Sync the schema received in the resize instruction.
c.logger.Printf("Holder ApplySchema")
c.logger.Debugf("holder applySchema")
if err := c.holder.applySchema(instr.Schema); err != nil {
return errors.Wrap(err, "applying schema")
}
@ -1297,7 +1295,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
// Stream shard from remote node.
c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI)
rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.Shard, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
// on the remote node. This occurs when a shard has been skipped and
@ -1651,17 +1649,17 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
switch e.Event {
case NodeJoin:
c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI)
c.logger.Debugf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI)
// Ignore the event if this is not the coordinator.
if !c.isCoordinator() {
return nil
}
return c.nodeJoin(e.Node)
case NodeLeave:
c.logger.Printf("received node leave on %s: %s, uri: %v", c.Node, e.Node, e.Node.URI)
c.mu.Lock()
defer c.mu.Unlock()
if c.unprotectedIsCoordinator() {
c.logger.Printf("received node leave: %v", e.Node)
// if removeNodeBasicSorted succeeds, that means that the node was
// not already removed by a removeNode request. We treat this as the
// host being temporarily unavailable, and expect it to come back
@ -1673,7 +1671,6 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
err = c.unprotectedSetStateAndBroadcast(c.determineClusterState())
}
}
c.logger.Printf("finished node leave on %s: %s, uri: %v", c.Node, e.Node, e.Node.URI)
case NodeUpdate:
c.logger.Printf("received node update event: id: %v, string: %v, uri: %v", e.Node.ID, e.Node.String(), e.Node.URI)
// NodeUpdate is intentionally not implemented.
@ -1686,7 +1683,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
func (c *cluster) nodeJoin(node *Node) error {
c.mu.Lock()
defer c.mu.Unlock()
c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID)
c.logger.Printf("node join event on coordinator, node: %s, id: %s", node.URI, node.ID)
if c.needTopologyAgreement() {
// A host that is not part of the topology can't be added to the STARTING cluster.
if !c.Topology.ContainsID(node.ID) {
@ -1726,7 +1723,7 @@ func (c *cluster) nodeJoin(node *Node) error {
// the cluster.
if cnode := c.unprotectedNodeByID(node.ID); cnode != nil {
if cnode.URI != node.URI {
c.logger.Printf("Node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI)
c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI)
cnode.URI = node.URI
}
return c.unprotectedSetStateAndBroadcast(c.determineClusterState())

5
cmd/import.go
View file

@ -19,10 +19,8 @@ import (
"io"
"github.com/pilosa/pilosa"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
"github.com/spf13/cobra"
)
var Importer *ctl.ImportCommand
@ -55,6 +53,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")

27
cmd/server.go
View file

@ -17,11 +17,13 @@ package cmd
import (
"io"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/tracing"
"github.com/pilosa/pilosa/tracing/opentracing"
"github.com/pkg/errors"
"github.com/spf13/cobra"
jaegercfg "github.com/uber/jaeger-client-go/config"
)
// Server is global so that tests can control and verify it.
@ -39,9 +41,28 @@ It will load existing data from the configured
directory and start listening for client connections
on the configured port.`,
RunE: func(cmd *cobra.Command, args []string) error {
// Start & run the server.
if err := Server.Start(); err != nil {
return errors.Wrap(err, "running server")
}
// Initialize tracing in the command since it is global.
var cfg jaegercfg.Configuration
cfg.ServiceName = "pilosa"
cfg.Sampler = &jaegercfg.SamplerConfig{
Type: Server.Config.Tracing.SamplerType,
Param: Server.Config.Tracing.SamplerParam,
}
cfg.Reporter = &jaegercfg.ReporterConfig{
LocalAgentHostPort: Server.Config.Tracing.AgentHostPort,
}
tracer, closer, err := cfg.NewTracer()
if err != nil {
return errors.Wrap(err, "initializing jaeger tracer")
}
defer closer.Close()
tracing.GlobalTracer = opentracing.NewTracer(tracer)
return errors.Wrap(Server.Wait(), "waiting on Server")
},
}

16
ctl/import.go
View file

@ -99,13 +99,15 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
cmd.client = client
if cmd.CreateSchema {
// set the correct type for the field
if cmd.FieldOptions.TimeQuantum != "" {
cmd.FieldOptions.Type = "time"
} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
cmd.FieldOptions.Type = "int"
} else {
cmd.FieldOptions.Type = "set"
if cmd.FieldOptions.Type == "" {
// set the correct type for the field
if cmd.FieldOptions.TimeQuantum != "" {
cmd.FieldOptions.Type = "time"
} else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 {
cmd.FieldOptions.Type = "int"
} else {
cmd.FieldOptions.Type = "set"
}
}
err := cmd.ensureSchema(ctx)
if err != nil {

5
ctl/server.go
View file

@ -68,4 +68,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
// Tracing
flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.")
flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
}

36
docs/configuration.md
View file

@ -307,6 +307,42 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
skip-verify = true
```
#### Tracing Sampler Type
* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote)
* Flag: `tracing.sampler-type`
* Env: `PILOSA_TRACING_SAMPLER_TYPE`
* Config:
```toml
[tracing]
sampler-type = "remote"
```
#### Tracing Sampler Parameter
* Description: Jaeger sampler parameter (number)
* Flag: `tracing.sampler-param`
* Env: `PILOSA_TRACING_SAMPLER_PARAM`
* Config:
```toml
[tracing]
sampler-param = 0.001
```
#### Tracing Agent Host/Port
* Description: Jaeger agent host:port
* Flag: `tracing.agent-host-port`
* Env: `PILOSA_TRACING_AGENT_HOST_PORT`
* Config:
```toml
[tracing]
agent-host-port = "localhost:6831"
```
#### Translation Map Size
* Description: Size in bytes of mmap to allocate for key translation

35
encoding/proto/proto.go
View file

@ -217,6 +217,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
decodeImportValueRequest(msg, mt)
return nil
case *pilosa.ImportRoaringRequest:
msg := &internal.ImportRoaringRequest{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling ImportRoaringRequest")
}
decodeImportRoaringRequest(msg, mt)
return nil
case *pilosa.ImportResponse:
msg := &internal.ImportResponse{}
err := proto.Unmarshal(buf, msg)
@ -292,6 +300,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
return encodeImportRequest(mt)
case *pilosa.ImportValueRequest:
return encodeImportValueRequest(mt)
case *pilosa.ImportRoaringRequest:
return encodeImportRoaringRequest(mt)
case *pilosa.ImportResponse:
return encodeImportResponse(mt)
case *pilosa.BlockDataRequest:
@ -348,6 +358,22 @@ func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValu
}
}
func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.ImportRoaringRequest {
views := make([]*internal.ImportRoaringRequestView, len(m.Views))
i := 0
for viewName, viewData := range m.Views {
views[i] = &internal.ImportRoaringRequestView{
Name: viewName,
Data: viewData,
}
i += 1
}
return &internal.ImportRoaringRequest{
Clear: m.Clear,
Views: views,
}
}
func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest {
return &internal.QueryRequest{
Query: m.Query,
@ -916,6 +942,15 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV
m.Values = pb.Values
}
func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {
views := map[string][]byte{}
for _, view := range pb.Views {
views[view.Name] = view.Data
}
m.Clear = pb.Clear
m.Views = views
}
func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) {
m.Err = pb.Err
}

166
executor.go
View file

@ -22,6 +22,7 @@ import (
"time"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
)
@ -81,7 +82,11 @@ func newExecutor(opts ...executorOption) *executor {
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
defer span.Finish()
resp := QueryResponse{}
// Verify that an index is set.
if index == "" {
return resp, ErrIndexRequired
@ -105,10 +110,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
// Translate query keys to ids, if necessary.
// No need to translate a remote call.
if !opt.Remote {
for i := range q.Calls {
if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
return resp, err
}
if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil {
return resp, err
}
}
@ -154,11 +157,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
// Translate response objects from ids to keys, if necessary.
// No need to translate a remote call.
if !opt.Remote {
for i := range results {
results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
if err != nil {
return resp, err
}
if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil {
return resp, err
}
}
@ -189,6 +189,9 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
}
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute")
defer span.Finish()
// Don't bother calculating shards for query types that don't require it.
needsShards := needsShards(q.Calls)
@ -225,6 +228,9 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
// executeCall executes a call.
func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall")
defer span.Finish()
if err := e.validateCallArgs(c); err != nil {
return nil, errors.Wrap(err, "validating args")
}
@ -292,6 +298,9 @@ func (e *executor) validateCallArgs(c *pql.Call) error {
}
func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeOptionsCall")
defer span.Finish()
optCopy := &execOptions{}
*optCopy = *opt
if arg, ok := c.Args["columnAttrs"]; ok {
@ -335,6 +344,9 @@ func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql.
// executeSum executes a Sum() call.
func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSum")
defer span.Finish()
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Sum(): field required")
}
@ -368,6 +380,9 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
// executeMin executes a Min() call.
func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin")
defer span.Finish()
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
@ -401,6 +416,9 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh
// executeMax executes a Max() call.
func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
defer span.Finish()
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
@ -434,6 +452,9 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh
// executeBitmapCall executes a call that returns a bitmap.
func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
defer span.Finish()
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeBitmapCallShard(ctx, index, c, shard)
@ -500,6 +521,9 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// executeBitmapCallShard executes a bitmap call for a single shard.
func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
switch c.Name {
case "Row":
return e.executeBitmapShard(ctx, index, c, shard)
@ -524,6 +548,9 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *
// executeSumCountShard calculates the sum and count for bsiGroups on a shard.
func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard")
defer span.Finish()
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
@ -562,6 +589,9 @@ func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pq
// executeMinShard calculates the min for bsiGroups on a shard.
func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinShard")
defer span.Finish()
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
@ -600,6 +630,9 @@ func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Cal
// executeMaxShard calculates the max for bsiGroups on a shard.
func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxShard")
defer span.Finish()
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
@ -640,6 +673,9 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN")
defer span.Finish()
idsArg, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
@ -679,6 +715,9 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
}
func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards")
defer span.Finish()
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeTopNShard(ctx, index, c, shard)
@ -704,6 +743,9 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C
// executeTopNShard executes a TopN call for a single shard.
func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard")
defer span.Finish()
field, _ := c.Args["_field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
@ -766,6 +808,9 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca
// executeDifferenceShard executes a difference() call for a local shard.
func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDifferenceShard")
defer span.Finish()
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Difference query is currently not supported")
@ -1095,8 +1140,11 @@ func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call
return frag.rows(start, filters...), nil
}
func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
// Fetch index.
func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapShard")
defer span.Finish()
// Fetch column label from index.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
@ -1128,6 +1176,9 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca
// executeIntersectShard executes a intersect() call for a local shard.
func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard")
defer span.Finish()
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Intersect query is currently not supported")
@ -1150,6 +1201,9 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
// executeRangeShard executes a range() call for a local shard.
func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeRangeShard")
defer span.Finish()
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
return e.executeBSIGroupRangeShard(ctx, index, c, shard)
@ -1222,7 +1276,10 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C
}
// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard.
func (e *executor) executeBSIGroupRangeShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBSIGroupRangeShard")
defer span.Finish()
// Only one conditional should be present.
if len(c.Args) == 0 {
return nil, errors.New("Range(): condition required")
@ -1353,6 +1410,9 @@ func (e *executor) executeBSIGroupRangeShard(_ context.Context, index string, c
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
defer span.Finish()
other := NewRow()
for i, input := range c.Children {
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
@ -1372,6 +1432,9 @@ func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.C
// executeXorShard executes a xor() call for a local shard.
func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeXorShard")
defer span.Finish()
other := NewRow()
for i, input := range c.Children {
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
@ -1391,6 +1454,9 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal
// executeNotShard executes a not() call for a local shard.
func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard")
defer span.Finish()
if len(c.Children) == 0 {
return nil, errors.New("Not() requires an input row")
} else if len(c.Children) > 1 {
@ -1439,6 +1505,9 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C
// executeCount executes a count() call.
func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
defer span.Finish()
if len(c.Children) == 0 {
return 0, errors.New("Count() requires an input bitmap")
} else if len(c.Children) > 1 {
@ -1471,6 +1540,9 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call,
// executeClearBit executes a Clear() call.
func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBit")
defer span.Finish()
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Clear() argument required: field")
@ -1506,6 +1578,9 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
// executeClearBitField executes a Clear() call for a field.
func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBitField")
defer span.Finish()
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.shardNodes(index, shard) {
@ -1536,6 +1611,9 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
// executeClearRow executes a ClearRow() call.
func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearRow")
defer span.Finish()
// Ensure the field type supports ClearRow().
fieldName, err := c.FieldArg()
if err != nil {
@ -1572,7 +1650,10 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal
}
// executeClearRowShard executes a ClearRow() call for a single shard.
func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeClearRowShard")
defer span.Finish()
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("ClearRow() argument required: field")
@ -1682,7 +1763,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.
if err != nil {
return false, errors.Wrap(err, "creating view")
}
fragment, err = view.createFragmentIfNotExists(shard)
fragment, err = view.CreateFragmentIfNotExists(shard)
if err != nil {
return false, errors.Wrapf(err, "creating fragment: %d", shard)
}
@ -1698,6 +1779,9 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSet")
defer span.Finish()
// Read colID.
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
@ -1765,6 +1849,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
// executeSetBitField executes a Set() call for a specific field.
func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetBitField")
defer span.Finish()
shard := colID / ShardWidth
ret := false
@ -1797,6 +1884,9 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
// executeSetValueField executes a Set() call for a specific int field.
func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetValueField")
defer span.Finish()
shard := colID / ShardWidth
ret := false
@ -1829,6 +1919,9 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetRowAttrs")
defer span.Finish()
fieldName, ok := c.Args["_field"].(string)
if !ok {
return errors.New("SetRowAttrs() field required")
@ -1886,6 +1979,9 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *execOptions) ([]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBulkSetRowAttrs")
defer span.Finish()
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
@ -1973,6 +2069,9 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// executeSetColumnAttrs executes a SetColumnAttrs() call.
func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetColumnAttrs")
defer span.Finish()
// Retrieve index.
idx := e.Holder.Index(index)
if idx == nil {
@ -2021,6 +2120,9 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
// remoteExec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64) (results []interface{}, err error) { // nolint: interfacer
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
defer span.Finish()
// Encode request object.
pbreq := &QueryRequest{
Query: q.String(),
@ -2059,6 +2161,9 @@ loop:
// If a mapping of shards to a node fails then the shards are resplit across
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapReduce")
defer span.Finish()
ch := make(chan mapResponse)
// Wrap context with a cancel to kill goroutines on exit.
@ -2073,7 +2178,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.nodes).Clone()
} else {
nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)}
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
@ -2118,6 +2223,9 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
}
func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
defer span.Finish()
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
@ -2153,6 +2261,9 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
// mapperLocal performs map & reduce entirely on the local node.
func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal")
defer span.Finish()
ch := make(chan mapResponse, len(shards))
for _, shard := range shards {
@ -2189,6 +2300,18 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
}
}
func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, calls []*pql.Call) error {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateCalls")
defer span.Finish()
for i := range calls {
if err := e.translateCall(index, idx, calls[i]); err != nil {
return err
}
}
return nil
}
func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
var colKey, rowKey, fieldName string
switch c.Name {
@ -2337,6 +2460,19 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e
return nil
}
func (e *executor) translateResults(ctx context.Context, index string, idx *Index, calls []*pql.Call, results []interface{}) (err error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults")
defer span.Finish()
for i := range results {
results[i], err = e.translateResult(index, idx, calls[i], results[i])
if err != nil {
return err
}
}
return nil
}
func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) {
switch result := result.(type) {
case *Row:

83
field.go
View file

@ -133,7 +133,9 @@ func OptFieldTypeInt(min, max int64) FieldOption {
}
}
func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
// OptFieldTypeTime sets the field type to time.
// Pass true to skip creation of the standard view.
func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
@ -143,6 +145,7 @@ func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
}
fo.Type = FieldTypeTime
fo.TimeQuantum = timeQuantum
fo.NoStandardView = len(opt) >= 1 && opt[0]
return nil
}
}
@ -448,6 +451,7 @@ func (f *Field) loadMeta() error {
f.options.Max = pb.Max
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
f.options.NoStandardView = pb.NoStandardView
return nil
}
@ -514,6 +518,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Min = 0
f.options.Max = 0
f.options.Keys = opt.Keys
f.options.NoStandardView = opt.NoStandardView
// Set the time quantum.
if err := f.setTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
@ -797,18 +802,19 @@ func (f *Field) Row(rowID uint64) (*Row, error) {
// SetBit sets a bit on a view within the field.
func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) {
viewName := viewStandard
if !f.options.NoStandardView {
// Retrieve view. Exit if it doesn't exist.
view, err := f.createViewIfNotExists(viewName)
if err != nil {
return changed, errors.Wrap(err, "creating view")
}
// Retrieve view. Exit if it doesn't exist.
view, err := f.createViewIfNotExists(viewName)
if err != nil {
return changed, errors.Wrap(err, "creating view")
}
// Set non-time bit.
if v, err := view.setBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "setting on view")
} else if v {
changed = v
// Set non-time bit.
if v, err := view.setBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "setting on view")
} else if v {
changed = v
}
}
// Exit early if no timestamp is specified.
@ -1092,9 +1098,11 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts
standard = []string{viewStandard}
} else {
standard = viewsByTime(viewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, viewStandard)
if !f.options.NoStandardView {
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, viewStandard)
}
}
// Attach bit to each standard view.
@ -1184,9 +1192,10 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
return nil
}
func (f *Field) importRoaring(data []byte, shard uint64, clear bool) error {
viewName := viewStandard
func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear bool) error {
if viewName == "" {
viewName = viewStandard
}
view, err := f.createViewIfNotExists(viewName)
if err != nil {
return errors.Wrap(err, "creating view")
@ -1225,13 +1234,14 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FieldOptions represents options to set when initializing a field.
type FieldOptions struct {
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
Keys bool `json:"keys"`
CacheSize uint32 `json:"cacheSize,omitempty"`
CacheType string `json:"cacheType,omitempty"`
Type string `json:"type,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
CacheType string `json:"cacheType,omitempty"`
Type string `json:"type,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
// applyDefaultOptions returns a new FieldOptions object
@ -1257,13 +1267,14 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
return nil
}
return &internal.FieldOptions{
Type: o.Type,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Min: o.Min,
Max: o.Max,
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
Type: o.Type,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Min: o.Min,
Max: o.Max,
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
NoStandardView: o.NoStandardView,
}
}
@ -1295,13 +1306,15 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
})
case FieldTypeTime:
return json.Marshal(struct {
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
NoStandardView bool `json:"noStandardView"`
}{
o.Type,
o.TimeQuantum,
o.Keys,
o.NoStandardView,
})
case FieldTypeMutex:
return json.Marshal(struct {

20
fragment.go
View file

@ -40,6 +40,7 @@ import (
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/stats"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
)
@ -1492,9 +1493,6 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor
lastRowID = rowID
rowSet[rowID] = struct{}{}
}
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
}
f.mu.Lock()
@ -1518,6 +1516,9 @@ func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *Impor
// Update cache counts for all affected rows.
for rowID := range rowSet {
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
n := results.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
f.cache.BulkAdd(rowID, n)
}
@ -1734,7 +1735,6 @@ func (f *fragment) snapshot() error {
// f.mu must be locked when calling it.
func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
start := time.Now()
defer track(start, completeMessage, f.stats, f.Logger)
@ -2188,6 +2188,9 @@ func (s *fragmentSyncer) isClosing() bool {
// syncFragment compares checksums for the local and remote fragments and
// then merges any blocks which have differences.
func (s *fragmentSyncer) syncFragment() error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "FragmentSyncer.syncFragment")
defer span.Finish()
// Determine replica set.
nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard)
if len(nodes) == 1 {
@ -2205,7 +2208,7 @@ func (s *fragmentSyncer) syncFragment() error {
}
// Retrieve remote blocks.
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.view, s.Fragment.shard)
blocks, err := s.Cluster.InternalClient.FragmentBlocks(ctx, &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.view, s.Fragment.shard)
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -2265,6 +2268,9 @@ func (s *fragmentSyncer) syncFragment() error {
// syncBlock sends and receives all rows for a given block.
// Returns an error if any remote hosts are unreachable.
func (s *fragmentSyncer) syncBlock(id int) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "FragmentSyncer.syncBlock")
defer span.Finish()
f := s.Fragment
// Read pairs from each remote block.
@ -2284,7 +2290,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
uris = append(uris, uri)
// Only sync the standard block.
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.view, f.shard, id)
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(ctx, &node.URI, f.index, f.field, f.view, f.shard, id)
if err != nil {
return errors.Wrap(err, "getting block")
}
@ -2346,7 +2352,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
Query: buffers[k].String(),
Remote: true,
}
_, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest)
_, err := s.Cluster.InternalClient.QueryNode(ctx, uris[i], f.index, queryRequest)
if err != nil {
return errors.Wrap(err, "executing")
}

17
fragment_internal_test.go
View file

@ -25,6 +25,8 @@ import (
"testing"
"testing/quick"
"golang.org/x/sync/errgroup"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/roaring"
@ -1399,6 +1401,21 @@ func TestFragment_ImportSet(t *testing.T) {
}
}
func TestFragment_ConcurrentImport(t *testing.T) {
t.Run("bulkImportStandard", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
eg := errgroup.Group{}
eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) })
eg.Go(func() error { return f.bulkImportStandard([]uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) })
err := eg.Wait()
if err != nil {
t.Fatalf("importing data to fragment: %v", err)
}
})
}
// Ensure a fragment can import mutually exclusive values.
func TestFragment_ImportMutex(t *testing.T) {
tests := []struct {

5
handler.go
View file

@ -96,6 +96,11 @@ type ImportRequest struct {
Timestamps []int64
}
type ImportRoaringRequest struct {
Clear bool
Views map[string][]byte
}
type ImportResponse struct {
Err string
}

51
holder.go
View file

@ -30,6 +30,7 @@ import (
"github.com/pilosa/pilosa/logger"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/stats"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -57,7 +58,7 @@ type Holder struct {
NewPrimaryTranslateStore func(interface{}) TranslateStore
// opened channel is closed once Open() completes.
opened chan struct{}
opened lockedChan
broadcaster broadcaster
@ -79,13 +80,39 @@ type Holder struct {
Logger logger.Logger
}
// lockedChan looks a little ridiculous admittedly, but exists for good reason.
// The channel within is used (for example) to signal to other goroutines when
// the Holder has finished opening (via closing the channel). However, it is
// possible for the holder to be closed and then reopened, but a channel which
// is closed cannot be re-opened. We must create a new channel - this creates a
// data race with any goroutine which might be accessing the channel. To ensure
// that there is no data race on the value of the channel itself, we wrap any
// operation on it with an RWMutex so that we can guarantee that nothing is
// trying to listen on it when it gets swapped.
type lockedChan struct {
ch chan struct{}
mu sync.RWMutex
}
func (lc *lockedChan) Close() {
lc.mu.RLock()
close(lc.ch)
lc.mu.RUnlock()
}
func (lc *lockedChan) Recv() {
lc.mu.RLock()
<-lc.ch
lc.mu.RUnlock()
}
// NewHolder returns a new instance of Holder.
func NewHolder() *Holder {
return &Holder{
indexes: make(map[string]*Index),
closing: make(chan struct{}),
opened: make(chan struct{}),
opened: lockedChan{ch: make(chan struct{})},
translateFile: NewTranslateFile(),
NewPrimaryTranslateStore: newNopTranslateStore,
@ -159,7 +186,7 @@ func (h *Holder) Open() error {
h.Stats.Open()
close(h.opened)
h.opened.Close()
return nil
}
@ -184,7 +211,9 @@ func (h *Holder) Close() error {
}
// Reset opened in case Holder needs to be reopened.
h.opened = make(chan struct{})
h.opened.mu.Lock()
h.opened.ch = make(chan struct{})
h.opened.mu.Unlock()
return nil
}
@ -474,7 +503,7 @@ func (h *Holder) flushCaches() {
}
if err := fragment.FlushCache(); err != nil {
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath())
h.Logger.Printf("ERROR flushing cache: err=%s, path=%s", err, fragment.cachePath())
}
}
}
@ -535,7 +564,7 @@ func (h *Holder) setFileLimit() {
h.Logger.Printf("ERROR checking open file limit: %s", err)
} else {
if oldLimit.Cur < fileLimit {
h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit)
h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/latest/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit)
}
}
}
@ -692,6 +721,9 @@ func (s *holderSyncer) SyncHolder() error {
// syncIndex synchronizes index attributes with the rest of the cluster.
func (s *holderSyncer) syncIndex(index string) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncIndex")
defer span.Finish()
// Retrieve index reference.
idx := s.Holder.Index(index)
if idx == nil {
@ -710,7 +742,7 @@ func (s *holderSyncer) syncIndex(index string) error {
for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks)
m, err := s.Cluster.InternalClient.ColumnAttrDiff(ctx, &node.URI, index, blks)
if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
@ -735,6 +767,9 @@ func (s *holderSyncer) syncIndex(index string) error {
// syncField synchronizes field attributes with the rest of the cluster.
func (s *holderSyncer) syncField(index, name string) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncField")
defer span.Finish()
// Retrieve field reference.
f := s.Holder.Field(index, name)
if f == nil {
@ -754,7 +789,7 @@ func (s *holderSyncer) syncField(index, name string) error {
for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks)
m, err := s.Cluster.InternalClient.RowAttrDiff(ctx, &node.URI, index, name, blks)
if err == ErrFieldNotFound {
continue // field not created remotely yet, skip
} else if err != nil {

127
http/client.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
)
@ -66,6 +67,8 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client)
// MaxShardByIndex returns the number of shards on a server by index.
func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.MaxShardByIndex")
defer span.Finish()
return c.maxShardByIndex(ctx)
}
@ -91,9 +94,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64
defer resp.Body.Close()
var rsp getShardsMaxResponse
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
@ -102,6 +103,9 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64
// Schema returns all index and field schema information.
func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema")
defer span.Finish()
// Execute request against the host.
u := c.defaultURI.Path("/schema")
@ -130,6 +134,9 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error
// CreateIndex creates a new index on the server.
func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex")
defer span.Finish()
// Encode query request.
buf, err := json.Marshal(&postIndexRequest{
Options: opt,
@ -152,7 +159,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
// Execute request against the host.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp.StatusCode == http.StatusConflict {
if resp != nil && resp.StatusCode == http.StatusConflict {
return pilosa.ErrIndexExists
}
return err
@ -162,6 +169,9 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
// FragmentNodes returns a list of nodes that own a shard.
func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentNodes")
defer span.Finish()
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes")
u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode()
@ -191,6 +201,9 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
// Nodes returns a list of all nodes.
func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Nodes")
defer span.Finish()
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/internal/nodes")
@ -219,11 +232,16 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) {
// Query executes query against the index.
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Query")
defer span.Finish()
return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode")
defer span.Finish()
if index == "" {
return nil, pilosa.ErrIndexRequired
} else if queryRequest.Query == "" {
@ -258,8 +276,6 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return nil, errors.New(string(body))
}
qresp := &pilosa.QueryResponse{}
@ -274,6 +290,9 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
// Import bulk imports bits for a single shard to a host.
func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit, opts ...pilosa.ImportOption) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Import")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
@ -321,6 +340,9 @@ func getCoordinatorNode(nodes []*pilosa.Node) *pilosa.Node {
// ImportK bulk imports bits specified by string keys to a host.
func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits []pilosa.Bit, opts ...pilosa.ImportOption) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportK")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
@ -361,6 +383,9 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits
}
func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureIndex")
defer span.Finish()
err := c.CreateIndex(ctx, name, options)
if err == nil || errors.Cause(err) == pilosa.ErrIndexExists {
return nil
@ -369,10 +394,14 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p
}
func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureField")
defer span.Finish()
return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{})
}
func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.EnsureFieldWithOptions")
defer span.Finish()
err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt)
if err == nil || errors.Cause(err) == pilosa.ErrFieldExists {
return nil
@ -408,6 +437,9 @@ func (c *InternalClient) marshalImportPayload(index, field string, shard uint64,
// importNode sends a pre-marshaled import request to a node.
func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode")
defer span.Finish()
// Create URL & HTTP request.
path := fmt.Sprintf("/index/%s/field/%s/import", index, field)
u := nodePathToURL(node, path)
@ -455,6 +487,9 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
// ImportValue bulk imports field values for a single shard to a host.
func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValue")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
@ -493,6 +528,9 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
// ImportValueK bulk imports keyed field values to a host.
func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue, opts ...pilosa.ImportOption) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportValueK")
defer span.Finish()
buf, err := c.marshalImportValuePayload(index, field, 0, vals)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
@ -550,7 +588,10 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui
// ImportRoaring does fast import of raw bits in roaring format (pilosa or
// official format, see API.ImportRoaring).
func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, data []byte, opts ...pilosa.ImportOption) error {
func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
@ -560,32 +601,27 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
uri = c.defaultURI
}
// Set up import options.
options := &pilosa.ImportOptions{}
for _, opt := range opts {
err := opt(options)
if err != nil {
return errors.Wrap(err, "applying option")
}
}
vals := url.Values{}
vals.Set("remote", strconv.FormatBool(remote))
if options.Clear {
vals.Set("clear", "true")
}
url := fmt.Sprintf("%s/index/%s/field/%s/import-roaring/%d?%s", uri, index, field, shard, vals.Encode())
// Marshal data to protobuf.
data, err := c.serializer.Marshal(req)
if err != nil {
return errors.Wrap(err, "marshal import request")
}
// Generate HTTP request.
req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/x-binary")
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(req.WithContext(ctx))
resp, err := c.executeRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}
@ -595,13 +631,16 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
rbody := &pilosa.ImportResponse{}
dec.Decode(rbody)
if rbody.Err != "" {
return errors.Errorf("importing roaring: %v", rbody.Err)
return errors.Wrap(errors.New(rbody.Err), "importing roaring")
}
return nil
}
// ExportCSV bulk exports data for a single shard from a host to CSV format.
func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
@ -632,6 +671,9 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha
// exportNode copies a CSV export from a node to w.
func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.exportNodeCSV")
defer span.Finish()
// Create URL.
u := nodePathToURL(node, "/export")
u.RawQuery = url.Values{
@ -664,6 +706,9 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i
}
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI")
defer span.Finish()
node := &pilosa.Node{
URI: uri,
}
@ -671,6 +716,9 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field
}
func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.backupShardNode")
defer span.Finish()
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
@ -689,7 +737,7 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp.StatusCode == http.StatusNotFound {
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, pilosa.ErrFragmentNotFound
}
return nil, err
@ -699,11 +747,16 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin
}
func (c *InternalClient) CreateField(ctx context.Context, index, field string) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateField")
defer span.Finish()
return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{})
}
// CreateField creates a new field on the server.
func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
}
@ -746,7 +799,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
// Execute request against the host.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp.StatusCode == http.StatusConflict {
if resp != nil && resp.StatusCode == http.StatusConflict {
return pilosa.ErrFieldExists
}
return err
@ -758,6 +811,9 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks")
defer span.Finish()
if uri == nil {
uri = c.defaultURI
}
@ -782,7 +838,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
// Return the appropriate error.
if resp.StatusCode == http.StatusNotFound {
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, pilosa.ErrFragmentNotFound
}
return nil, err
@ -799,6 +855,9 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
// BlockData returns row/column id pairs for a block.
func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.BlockData")
defer span.Finish()
if uri == nil {
panic("need to pass a URI to BlockData")
}
@ -825,7 +884,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp.StatusCode == http.StatusNotFound {
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, nil, nil
}
return nil, nil, err
@ -844,6 +903,9 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
// ColumnAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ColumnAttrDiff")
defer span.Finish()
if uri == nil {
uri = c.defaultURI
}
@ -881,6 +943,9 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RowAttrDiff")
defer span.Finish()
if uri == nil {
uri = c.defaultURI
}
@ -904,7 +969,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp.StatusCode == http.StatusNotFound {
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, pilosa.ErrFieldNotFound
}
return nil, err
@ -921,6 +986,9 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
// SendMessage posts a message synchronously.
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.SendMessage")
defer span.Finish()
u := uriPathToURL(uri, "/internal/cluster/message")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
if err != nil {
@ -937,6 +1005,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
// executeRequest executes the given request and checks the Response
func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) {
tracing.GlobalTracer.InjectHTTPHeaders(req)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, errors.Wrap(err, "executing request")

30
http/client_test.go
View file

@ -408,8 +408,9 @@ func TestClient_ImportRoaring(t *testing.T) {
// Send import request.
host := cluster[0].URL()
c := MustNewClient(host, http.GetHTTPClient(nil))
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringData); err != nil {
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
@ -432,8 +433,9 @@ func TestClient_ImportRoaring(t *testing.T) {
}
// Ensure that sending a roaring import with the clear flag works as expected.
roaringDataClear, _ := hex.DecodeString("3A30000001000000010001001000000003000400") // [65539, 65540]
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil {
// [65539, 65540]
roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400")
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
@ -454,8 +456,9 @@ func TestClient_ImportRoaring(t *testing.T) {
}
// Ensure that sending a roaring import with the clear flag works as expected.
roaringDataClear, _ = hex.DecodeString("3A300000020000000000010001000100180000001C0000000400060001000300") // [4, 6, 65537, 65539]
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil {
// [4, 6, 65537, 65539]
roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300")
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
@ -476,8 +479,9 @@ func TestClient_ImportRoaring(t *testing.T) {
}
// Ensure that sending a roaring import with the clear flag works as expected.
roaringDataClear, _ = hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil {
// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100")
if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
t.Fatal(err)
}
@ -981,3 +985,13 @@ func MustNewClient(host string, h *gohttp.Client) *Client {
}
return &Client{InternalClient: c}
}
func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaringRequest {
roaringData, _ := hex.DecodeString(viewData)
return &pilosa.ImportRoaringRequest{
Clear: clear,
Views: map[string][]byte{
"": roaringData,
},
}
}

53
http/handler.go
View file

@ -24,9 +24,8 @@ import (
"io/ioutil"
"net"
"net/http"
"net/url"
// Imported for its side-effect of registering pprof endpoints with the server.
_ "net/http/pprof"
"net/url" // Imported for its side-effect of registering pprof endpoints with the server.
"reflect"
"runtime/debug"
"strconv"
@ -37,7 +36,7 @@ import (
"github.com/gorilla/mux"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/logger"
"github.com/pilosa/pilosa/tracing"
"github.com/pkg/errors"
)
@ -222,6 +221,15 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
})
}
func (h *Handler) extractTracing(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
span, ctx := tracing.GlobalTracer.ExtractHTTPHeaders(r)
defer span.Finish()
next.ServeHTTP(w, r.WithContext(ctx))
})
}
// newRouter creates a new mux http router.
func newRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
@ -262,6 +270,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
router.Use(handler.queryArgValidator)
router.Use(handler.extractTracing)
return router
}
@ -707,7 +716,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
case pilosa.FieldTypeInt:
fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
case pilosa.FieldTypeTime:
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum, req.Options.NoStandardView))
case pilosa.FieldTypeMutex:
fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize))
case pilosa.FieldTypeBool:
@ -730,13 +739,14 @@ type postFieldRequest struct {
// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values,
// and used for input validation.
type fieldOptions struct {
Type string `json:"type,omitempty"`
CacheType *string `json:"cacheType,omitempty"`
CacheSize *uint32 `json:"cacheSize,omitempty"`
Min *int64 `json:"min,omitempty"`
Max *int64 `json:"max,omitempty"`
TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
Keys *bool `json:"keys,omitempty"`
Type string `json:"type,omitempty"`
CacheType *string `json:"cacheType,omitempty"`
CacheSize *uint32 `json:"cacheSize,omitempty"`
Min *int64 `json:"min,omitempty"`
Max *int64 `json:"max,omitempty"`
TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
Keys *bool `json:"keys,omitempty"`
NoStandardView bool `json:"noStandardView,omitempty"`
}
func (o *fieldOptions) validate() error {
@ -1497,10 +1507,18 @@ func GetHTTPClient(t *tls.Config) *http.Client {
// handlPostRoaringImport
func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Content-Type") != "application/x-binary" {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
q := r.URL.Query()
remoteStr := q.Get("remote")
var remote bool
@ -1508,9 +1526,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
remote = true
}
// If the clear flag is true, treat the import as clear bits.
doClear := q.Get("clear") == "true"
// Read entire body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
@ -1518,6 +1533,12 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
return
}
req := &pilosa.ImportRoaringRequest{}
if err := h.api.Serializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
urlVars := mux.Vars(r)
shard, err := strconv.ParseUint(urlVars["shard"], 10, 64)
if err != nil {
@ -1527,7 +1548,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
resp := &pilosa.ImportResponse{}
// TODO give meaningful stats for import
err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body, pilosa.OptImportOptionsClear(doClear))
err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, remote, req)
if err != nil {
resp.Err = err.Error()
if _, ok := err.(pilosa.BadRequestError); ok {

16
index_test.go
View file

@ -70,6 +70,22 @@ func TestIndex_CreateField(t *testing.T) {
})
})
// Ensure time quantum can be set appropriately on a new field.
t.Run("TimeQuantumNoStandardView", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create field with explicit quantum with no standard view
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), true))
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected field time quantum: %s", q)
}
})
})
// Ensure field can include range columns.
t.Run("BSIFields", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {

249
internal/private.pb.go
View file

@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} }
func (m *IndexMeta) String() string { return proto.CompactTextString(m) }
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{0}
return fileDescriptor_private_08d4c0c27f7a355f, []int{0}
}
func (m *IndexMeta) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -83,6 +83,7 @@ type FieldOptions struct {
Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"`
NoStandardView bool `protobuf:"varint,12,opt,name=NoStandardView,proto3" json:"NoStandardView,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -92,7 +93,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} }
func (m *FieldOptions) String() string { return proto.CompactTextString(m) }
func (*FieldOptions) ProtoMessage() {}
func (*FieldOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{1}
return fileDescriptor_private_08d4c0c27f7a355f, []int{1}
}
func (m *FieldOptions) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -170,6 +171,13 @@ func (m *FieldOptions) GetKeys() bool {
return false
}
func (m *FieldOptions) GetNoStandardView() bool {
if m != nil {
return m.NoStandardView
}
return false
}
type ImportResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -181,7 +189,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} }
func (m *ImportResponse) String() string { return proto.CompactTextString(m) }
func (*ImportResponse) ProtoMessage() {}
func (*ImportResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{2}
return fileDescriptor_private_08d4c0c27f7a355f, []int{2}
}
func (m *ImportResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -232,7 +240,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} }
func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) }
func (*BlockDataRequest) ProtoMessage() {}
func (*BlockDataRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{3}
return fileDescriptor_private_08d4c0c27f7a355f, []int{3}
}
func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -308,7 +316,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} }
func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) }
func (*BlockDataResponse) ProtoMessage() {}
func (*BlockDataResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{4}
return fileDescriptor_private_08d4c0c27f7a355f, []int{4}
}
func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -362,7 +370,7 @@ func (m *Cache) Reset() { *m = Cache{} }
func (m *Cache) String() string { return proto.CompactTextString(m) }
func (*Cache) ProtoMessage() {}
func (*Cache) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{5}
return fileDescriptor_private_08d4c0c27f7a355f, []int{5}
}
func (m *Cache) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -409,7 +417,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} }
func (m *MaxShards) String() string { return proto.CompactTextString(m) }
func (*MaxShards) ProtoMessage() {}
func (*MaxShards) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{6}
return fileDescriptor_private_08d4c0c27f7a355f, []int{6}
}
func (m *MaxShards) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -458,7 +466,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} }
func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) }
func (*CreateShardMessage) ProtoMessage() {}
func (*CreateShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{7}
return fileDescriptor_private_08d4c0c27f7a355f, []int{7}
}
func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -519,7 +527,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} }
func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexMessage) ProtoMessage() {}
func (*DeleteIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{8}
return fileDescriptor_private_08d4c0c27f7a355f, []int{8}
}
func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -567,7 +575,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} }
func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) }
func (*CreateIndexMessage) ProtoMessage() {}
func (*CreateIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{9}
return fileDescriptor_private_08d4c0c27f7a355f, []int{9}
}
func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -623,7 +631,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} }
func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) }
func (*CreateFieldMessage) ProtoMessage() {}
func (*CreateFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{10}
return fileDescriptor_private_08d4c0c27f7a355f, []int{10}
}
func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -685,7 +693,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} }
func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteFieldMessage) ProtoMessage() {}
func (*DeleteFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{11}
return fileDescriptor_private_08d4c0c27f7a355f, []int{11}
}
func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -741,7 +749,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar
func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteAvailableShardMessage) ProtoMessage() {}
func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{12}
return fileDescriptor_private_08d4c0c27f7a355f, []int{12}
}
func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -804,7 +812,7 @@ func (m *Field) Reset() { *m = Field{} }
func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (*Field) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{13}
return fileDescriptor_private_08d4c0c27f7a355f, []int{13}
}
func (m *Field) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -865,7 +873,7 @@ func (m *Schema) Reset() { *m = Schema{} }
func (m *Schema) String() string { return proto.CompactTextString(m) }
func (*Schema) ProtoMessage() {}
func (*Schema) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{14}
return fileDescriptor_private_08d4c0c27f7a355f, []int{14}
}
func (m *Schema) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -913,7 +921,7 @@ func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{15}
return fileDescriptor_private_08d4c0c27f7a355f, []int{15}
}
func (m *Index) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -969,7 +977,7 @@ func (m *URI) Reset() { *m = URI{} }
func (m *URI) String() string { return proto.CompactTextString(m) }
func (*URI) ProtoMessage() {}
func (*URI) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{16}
return fileDescriptor_private_08d4c0c27f7a355f, []int{16}
}
func (m *URI) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1033,7 +1041,7 @@ func (m *Node) Reset() { *m = Node{} }
func (m *Node) String() string { return proto.CompactTextString(m) }
func (*Node) ProtoMessage() {}
func (*Node) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{17}
return fileDescriptor_private_08d4c0c27f7a355f, []int{17}
}
func (m *Node) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1102,7 +1110,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} }
func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) }
func (*NodeStateMessage) ProtoMessage() {}
func (*NodeStateMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{18}
return fileDescriptor_private_08d4c0c27f7a355f, []int{18}
}
func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1157,7 +1165,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} }
func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) }
func (*NodeEventMessage) ProtoMessage() {}
func (*NodeEventMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{19}
return fileDescriptor_private_08d4c0c27f7a355f, []int{19}
}
func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1213,7 +1221,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} }
func (m *NodeStatus) String() string { return proto.CompactTextString(m) }
func (*NodeStatus) ProtoMessage() {}
func (*NodeStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{20}
return fileDescriptor_private_08d4c0c27f7a355f, []int{20}
}
func (m *NodeStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1275,7 +1283,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} }
func (m *IndexStatus) String() string { return proto.CompactTextString(m) }
func (*IndexStatus) ProtoMessage() {}
func (*IndexStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{21}
return fileDescriptor_private_08d4c0c27f7a355f, []int{21}
}
func (m *IndexStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1330,7 +1338,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} }
func (m *FieldStatus) String() string { return proto.CompactTextString(m) }
func (*FieldStatus) ProtoMessage() {}
func (*FieldStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{22}
return fileDescriptor_private_08d4c0c27f7a355f, []int{22}
}
func (m *FieldStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1386,7 +1394,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} }
func (m *ClusterStatus) String() string { return proto.CompactTextString(m) }
func (*ClusterStatus) ProtoMessage() {}
func (*ClusterStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{23}
return fileDescriptor_private_08d4c0c27f7a355f, []int{23}
}
func (m *ClusterStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1450,7 +1458,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} }
func (m *BSIGroup) String() string { return proto.CompactTextString(m) }
func (*BSIGroup) ProtoMessage() {}
func (*BSIGroup) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{24}
return fileDescriptor_private_08d4c0c27f7a355f, []int{24}
}
func (m *BSIGroup) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1520,7 +1528,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} }
func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) }
func (*CreateViewMessage) ProtoMessage() {}
func (*CreateViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{25}
return fileDescriptor_private_08d4c0c27f7a355f, []int{25}
}
func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1583,7 +1591,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} }
func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteViewMessage) ProtoMessage() {}
func (*DeleteViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{26}
return fileDescriptor_private_08d4c0c27f7a355f, []int{26}
}
func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1649,7 +1657,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} }
func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) }
func (*ResizeInstruction) ProtoMessage() {}
func (*ResizeInstruction) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{27}
return fileDescriptor_private_08d4c0c27f7a355f, []int{27}
}
func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1735,7 +1743,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} }
func (m *ResizeSource) String() string { return proto.CompactTextString(m) }
func (*ResizeSource) ProtoMessage() {}
func (*ResizeSource) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{28}
return fileDescriptor_private_08d4c0c27f7a355f, []int{28}
}
func (m *ResizeSource) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1812,7 +1820,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp
func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) }
func (*ResizeInstructionComplete) ProtoMessage() {}
func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{29}
return fileDescriptor_private_08d4c0c27f7a355f, []int{29}
}
func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1873,7 +1881,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} }
func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*SetCoordinatorMessage) ProtoMessage() {}
func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{30}
return fileDescriptor_private_08d4c0c27f7a355f, []int{30}
}
func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1920,7 +1928,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa
func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*UpdateCoordinatorMessage) ProtoMessage() {}
func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{31}
return fileDescriptor_private_08d4c0c27f7a355f, []int{31}
}
func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1968,7 +1976,7 @@ func (m *Topology) Reset() { *m = Topology{} }
func (m *Topology) String() string { return proto.CompactTextString(m) }
func (*Topology) ProtoMessage() {}
func (*Topology) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{32}
return fileDescriptor_private_08d4c0c27f7a355f, []int{32}
}
func (m *Topology) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2021,7 +2029,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} }
func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) }
func (*RecalculateCaches) ProtoMessage() {}
func (*RecalculateCaches) Descriptor() ([]byte, []int) {
return fileDescriptor_private_725d4d7695f6ae76, []int{33}
return fileDescriptor_private_08d4c0c27f7a355f, []int{33}
}
func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2186,6 +2194,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.NoStandardView {
dAtA[i] = 0x60
i++
if m.NoStandardView {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if m.XXX_unrecognized != nil {
i += copy(dAtA[i:], m.XXX_unrecognized)
}
@ -3556,6 +3574,9 @@ func (m *FieldOptions) Size() (n int) {
if m.Keys {
n += 2
}
if m.NoStandardView {
n += 2
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4579,6 +4600,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
}
}
m.Keys = bool(v != 0)
case 12:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field NoStandardView", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.NoStandardView = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8836,79 +8877,79 @@ var (
ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_725d4d7695f6ae76) }
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_08d4c0c27f7a355f) }
var fileDescriptor_private_725d4d7695f6ae76 = []byte{
// 1121 bytes of a gzipped FileDescriptorProto
var fileDescriptor_private_08d4c0c27f7a355f = []byte{
// 1131 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5,
0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51,
0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c,
0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7,
0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66,
0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74,
0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29,
0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c,
0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e,
0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b,
0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1,
0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76,
0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d,
0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34,
0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88,
0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91,
0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7,
0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82,
0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42,
0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8,
0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43,
0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa,
0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77,
0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce,
0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a,
0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27,
0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03,
0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75,
0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25,
0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21,
0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f,
0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf,
0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e,
0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58,
0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce,
0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f,
0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14,
0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34,
0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4,
0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14,
0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b,
0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28,
0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99,
0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34,
0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a,
0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91,
0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24,
0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19,
0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e,
0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0,
0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4,
0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70,
0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1,
0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56,
0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43,
0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca,
0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68,
0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb,
0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a,
0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0,
0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a,
0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34,
0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17,
0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2,
0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84,
0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26,
0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1,
0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75,
0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc,
0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00,
0x00,
0x20, 0xf1, 0x02, 0x88, 0x27, 0xe2, 0x92, 0x47, 0xa8, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xb3, 0xbb,
0x76, 0x1c, 0x12, 0x05, 0xee, 0xe6, 0xfc, 0xce, 0x99, 0xf3, 0x7d, 0xce, 0xec, 0x42, 0x77, 0x9c,
0xc5, 0x27, 0x4c, 0xf2, 0x8d, 0x71, 0x26, 0xa4, 0x08, 0x5a, 0x71, 0x2a, 0x79, 0x96, 0xb2, 0x84,
0x3c, 0x87, 0x76, 0x94, 0x0e, 0xf9, 0xd9, 0x0e, 0x97, 0x2c, 0x08, 0xc0, 0x7f, 0xc1, 0x27, 0x79,
0xe8, 0xad, 0x39, 0xeb, 0x2d, 0x8a, 0xe7, 0xe0, 0x03, 0x58, 0xdc, 0xcb, 0xd8, 0xe0, 0x78, 0xfb,
0x2c, 0xce, 0x25, 0x4f, 0x07, 0x3c, 0xf4, 0x91, 0x3b, 0x83, 0x92, 0x37, 0x0e, 0xdc, 0xfa, 0x2a,
0xe6, 0xc9, 0xf0, 0xbb, 0xb1, 0x8c, 0x45, 0x9a, 0x07, 0xef, 0x42, 0x7b, 0x8b, 0x0d, 0x8e, 0xf8,
0xde, 0x64, 0xcc, 0x51, 0x63, 0x9b, 0x56, 0x40, 0xc9, 0xed, 0xc7, 0xaf, 0xb5, 0xc6, 0x2e, 0xad,
0x80, 0x60, 0x0d, 0x3a, 0x7b, 0xf1, 0x88, 0x7f, 0x5f, 0xb0, 0x54, 0x16, 0xa3, 0xb0, 0x81, 0xb7,
0xeb, 0x90, 0x72, 0x15, 0x15, 0xb7, 0x90, 0x85, 0xe7, 0x60, 0x09, 0xbc, 0x9d, 0x38, 0x0d, 0xdb,
0x6b, 0xce, 0xba, 0x47, 0xd5, 0x11, 0x11, 0x76, 0x16, 0x82, 0x41, 0xd8, 0x59, 0x19, 0x62, 0x67,
0x3a, 0xc4, 0x5d, 0xd1, 0x97, 0x2c, 0x1d, 0xb2, 0x6c, 0x78, 0x10, 0xf3, 0xd3, 0xf0, 0x96, 0x0e,
0x71, 0x1a, 0x25, 0x04, 0x16, 0xa3, 0xd1, 0x58, 0x64, 0x92, 0xf2, 0x7c, 0x2c, 0xd2, 0x1c, 0x2d,
0x6e, 0x67, 0x59, 0xe8, 0xa0, 0x13, 0xea, 0x48, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0x0c, 0x8e, 0x7b,
0x4c, 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0xff, 0x43, 0x03, 0x73, 0x6c, 0xe4, 0x34, 0xa1,
0x50, 0xcc, 0x57, 0xe8, 0x6a, 0x14, 0x09, 0x85, 0xe2, 0x7d, 0xcc, 0x98, 0x4f, 0x35, 0xa1, 0xd0,
0xfe, 0x11, 0xcb, 0x86, 0x98, 0x29, 0x9f, 0x6a, 0x42, 0xc5, 0x82, 0xde, 0xea, 0xf4, 0xe0, 0x99,
0x44, 0xb0, 0x5c, 0xb3, 0x6f, 0xdc, 0x5c, 0x81, 0x26, 0x15, 0xa7, 0x51, 0x2f, 0x0f, 0x9d, 0x35,
0x6f, 0xdd, 0xa7, 0x86, 0xc2, 0x22, 0x88, 0xa4, 0x18, 0xa5, 0x8a, 0xe5, 0x22, 0xab, 0x02, 0xc8,
0x3d, 0x68, 0x60, 0x45, 0x54, 0x94, 0xd5, 0x5d, 0x75, 0x24, 0x3f, 0x3b, 0xd0, 0xde, 0x61, 0x67,
0xe8, 0x46, 0x1e, 0x3c, 0x85, 0x96, 0xcd, 0x13, 0x0a, 0x75, 0x1e, 0xbf, 0xbf, 0x61, 0x1b, 0x6c,
0xa3, 0x14, 0xdb, 0xb0, 0x32, 0xdb, 0xa9, 0xcc, 0x26, 0xb4, 0xbc, 0xf2, 0xf6, 0xe7, 0xd0, 0x9d,
0x62, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c,
0x73, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x03, 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47,
0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x79, 0xc6, 0x75, 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a,
0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, 0x37, 0xd3, 0xf1, 0x0f, 0x7a, 0x49,
0xdf, 0xfa, 0x70, 0xb5, 0x6c, 0xf0, 0x00, 0x7c, 0x35, 0x6a, 0xe8, 0x42, 0xe7, 0xf1, 0x9d, 0x2a,
0x4f, 0xe5, 0x14, 0x52, 0x14, 0x20, 0x89, 0x55, 0x8a, 0xfe, 0x5c, 0x19, 0xd8, 0x9c, 0x56, 0x7a,
0x68, 0x4c, 0x79, 0x68, 0x6a, 0xa5, 0x32, 0x55, 0x1f, 0x53, 0x63, 0xed, 0x99, 0x0d, 0xf7, 0xa6,
0xd6, 0xc8, 0x00, 0xde, 0xd1, 0x1a, 0xbe, 0x3c, 0x61, 0x71, 0xc2, 0x0e, 0x93, 0x6b, 0x56, 0x64,
0x8e, 0xe3, 0x21, 0x2c, 0xe0, 0xdd, 0xa8, 0x67, 0xa6, 0xc0, 0x92, 0xe4, 0x07, 0x23, 0xaf, 0x5a,
0x7f, 0x97, 0x8d, 0xb8, 0xd1, 0x86, 0xe7, 0x32, 0x5e, 0xf7, 0xea, 0x78, 0x95, 0x61, 0x35, 0x2e,
0x6a, 0xd5, 0x79, 0xca, 0x30, 0x12, 0xe4, 0x09, 0x34, 0xfb, 0x83, 0x23, 0x3e, 0x62, 0xc1, 0x87,
0xb0, 0x80, 0x1e, 0xf2, 0xdc, 0x74, 0xf4, 0xed, 0x99, 0x4a, 0x51, 0xcb, 0x27, 0x3d, 0x13, 0xd9,
0x5c, 0x9f, 0x1e, 0x40, 0x13, 0xad, 0xe7, 0xa1, 0x3f, 0xab, 0x06, 0x71, 0x6a, 0xd8, 0x64, 0x1b,
0xbc, 0x7d, 0x1a, 0xa9, 0x49, 0x45, 0x0f, 0xac, 0x16, 0x43, 0x29, 0xdd, 0x5f, 0x8b, 0x5c, 0x9a,
0x3c, 0xe1, 0x59, 0x61, 0x2f, 0x45, 0x26, 0x31, 0x47, 0x5d, 0x8a, 0x67, 0x92, 0x83, 0xbf, 0x2b,
0x86, 0x3c, 0x58, 0x04, 0x37, 0xea, 0x19, 0x1d, 0x6e, 0xd4, 0x0b, 0xde, 0x43, 0xf5, 0x26, 0x35,
0xdd, 0xca, 0x89, 0x7d, 0x1a, 0x51, 0x34, 0x7c, 0x1f, 0xba, 0x51, 0xbe, 0x25, 0x44, 0x36, 0x8c,
0x53, 0x26, 0x45, 0x66, 0xde, 0x80, 0x69, 0x10, 0x27, 0x48, 0x32, 0xa9, 0x37, 0x76, 0x9b, 0x6a,
0x82, 0x3c, 0x83, 0x25, 0x65, 0x14, 0x09, 0x5b, 0xef, 0x15, 0x68, 0x2a, 0xac, 0x74, 0xc2, 0x50,
0x95, 0x06, 0xb7, 0xae, 0xe1, 0x5b, 0xad, 0x61, 0xfb, 0x84, 0xa7, 0xb2, 0xd6, 0x31, 0x48, 0xa3,
0x82, 0x2e, 0xd5, 0x44, 0x40, 0x74, 0x80, 0x26, 0x92, 0xc5, 0x2a, 0x12, 0x85, 0x52, 0xe4, 0x91,
0x5f, 0x1d, 0x00, 0xeb, 0x50, 0x91, 0x97, 0x57, 0x9c, 0xcb, 0xaf, 0x04, 0xeb, 0xb6, 0xf2, 0x66,
0x5a, 0x96, 0x2a, 0x29, 0x8d, 0x53, 0xdb, 0x19, 0x1f, 0x57, 0x9d, 0xa1, 0x4b, 0x7a, 0x77, 0xa6,
0x33, 0xb4, 0xd5, 0xaa, 0x3f, 0x5e, 0x42, 0xa7, 0x86, 0xcf, 0xed, 0x92, 0x8f, 0xca, 0x2e, 0x71,
0x67, 0x55, 0x22, 0x6e, 0x54, 0xda, 0x5e, 0x79, 0x01, 0x9d, 0x1a, 0x3c, 0x57, 0xe3, 0x3a, 0xdc,
0x9e, 0x9e, 0x43, 0xbb, 0xdf, 0x67, 0x61, 0x12, 0x43, 0x77, 0x2b, 0x29, 0x72, 0xc9, 0x33, 0xa3,
0x4e, 0x3d, 0x0a, 0x1a, 0x28, 0x8b, 0x57, 0x01, 0xf3, 0xeb, 0x17, 0xdc, 0x87, 0x86, 0x4a, 0xa3,
0x1e, 0xa7, 0x8b, 0x39, 0xd6, 0x4c, 0x72, 0x00, 0xad, 0xcd, 0x7e, 0xf4, 0x3c, 0x13, 0xc5, 0x78,
0xae, 0xd3, 0xf6, 0x4d, 0x77, 0x2f, 0xbe, 0xe9, 0xde, 0x85, 0x37, 0xdd, 0x2f, 0xdf, 0x74, 0xd2,
0x87, 0x65, 0xbd, 0x2a, 0xd5, 0x14, 0xdf, 0x64, 0xe1, 0xd8, 0x87, 0xd4, 0xab, 0x3d, 0xa4, 0x7d,
0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x54, 0xfa, 0xbb, 0x0b, 0xcb, 0x94, 0xe7, 0xf1, 0x6b, 0x1e, 0xa5,
0xb9, 0xcc, 0x8a, 0x81, 0xda, 0x49, 0xea, 0xfe, 0x37, 0xe2, 0xd0, 0x64, 0xdb, 0xa3, 0x9a, 0xb8,
0x4e, 0xa7, 0x07, 0x8f, 0xa0, 0x33, 0x3b, 0xb3, 0x17, 0x45, 0xeb, 0x22, 0xc1, 0x23, 0x58, 0xe8,
0x8b, 0x22, 0x1b, 0x94, 0xed, 0x5b, 0xdb, 0x93, 0xda, 0x33, 0xcd, 0xa6, 0x56, 0xac, 0x36, 0x1a,
0x8d, 0x2b, 0x46, 0xe3, 0xe9, 0x4c, 0x2b, 0x85, 0x4d, 0xbc, 0xf0, 0x56, 0x75, 0x61, 0x8a, 0x4d,
0xa7, 0xa5, 0xc9, 0x2f, 0x0e, 0xdc, 0xaa, 0xbb, 0x70, 0xad, 0xc1, 0x2d, 0x2b, 0xe2, 0xce, 0xad,
0x88, 0x37, 0xaf, 0x22, 0x7e, 0x55, 0x91, 0xea, 0x9b, 0xa0, 0x51, 0xfb, 0x26, 0x20, 0xc7, 0x70,
0xef, 0x42, 0x99, 0xb6, 0xc4, 0x68, 0xac, 0xfa, 0xe1, 0x5f, 0x94, 0x4b, 0xad, 0xb4, 0x2c, 0x33,
0x85, 0x6a, 0x53, 0x4d, 0x90, 0x4f, 0xe1, 0x6e, 0x9f, 0xcb, 0x5a, 0x91, 0x6c, 0xb7, 0xad, 0x81,
0xb7, 0xcb, 0x4f, 0x2f, 0x09, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xee, 0x8f, 0x87, 0x4c, 0xf2, 0x1b,
0xdd, 0xde, 0x84, 0xd6, 0x9e, 0x18, 0x8b, 0x44, 0xbc, 0x9a, 0x5c, 0x31, 0xf5, 0x21, 0x2c, 0xe8,
0xfd, 0xad, 0xd7, 0x48, 0x9b, 0x5a, 0x92, 0xdc, 0x51, 0x0d, 0x3d, 0x60, 0xc9, 0xa0, 0x48, 0x94,
0x1b, 0xea, 0x7b, 0x31, 0xdf, 0x5c, 0xfa, 0xe3, 0x7c, 0xd5, 0xf9, 0xf3, 0x7c, 0xd5, 0x79, 0x73,
0xbe, 0xea, 0xfc, 0xf6, 0xd7, 0xea, 0xff, 0x0e, 0x9b, 0xf8, 0xdf, 0xf1, 0xe4, 0xef, 0x00, 0x00,
0x00, 0xff, 0xff, 0x61, 0x80, 0xe4, 0xef, 0x88, 0x0c, 0x00, 0x00,
}

1
internal/private.proto
View file

@ -15,6 +15,7 @@ message FieldOptions {
int64 Max = 10;
string TimeQuantum = 5;
bool Keys = 11;
bool NoStandardView = 12;
}
message ImportResponse {

584
internal/public.pb.go
View file

@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} }
func (m *Row) String() string { return proto.CompactTextString(m) }
func (*Row) ProtoMessage() {}
func (*Row) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{0}
return fileDescriptor_public_5eafd62083455670, []int{0}
}
func (m *Row) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} }
func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) }
func (*RowIdentifiers) ProtoMessage() {}
func (*RowIdentifiers) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{1}
return fileDescriptor_public_5eafd62083455670, []int{1}
}
func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} }
func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{2}
return fileDescriptor_public_5eafd62083455670, []int{2}
}
func (m *Pair) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -215,7 +215,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} }
func (m *FieldRow) String() string { return proto.CompactTextString(m) }
func (*FieldRow) ProtoMessage() {}
func (*FieldRow) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{3}
return fileDescriptor_public_5eafd62083455670, []int{3}
}
func (m *FieldRow) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -270,7 +270,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} }
func (m *GroupCount) String() string { return proto.CompactTextString(m) }
func (*GroupCount) ProtoMessage() {}
func (*GroupCount) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{4}
return fileDescriptor_public_5eafd62083455670, []int{4}
}
func (m *GroupCount) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -325,7 +325,7 @@ func (m *ValCount) Reset() { *m = ValCount{} }
func (m *ValCount) String() string { return proto.CompactTextString(m) }
func (*ValCount) ProtoMessage() {}
func (*ValCount) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{5}
return fileDescriptor_public_5eafd62083455670, []int{5}
}
func (m *ValCount) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -381,7 +381,7 @@ func (m *Bit) Reset() { *m = Bit{} }
func (m *Bit) String() string { return proto.CompactTextString(m) }
func (*Bit) ProtoMessage() {}
func (*Bit) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{6}
return fileDescriptor_public_5eafd62083455670, []int{6}
}
func (m *Bit) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -444,7 +444,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} }
func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) }
func (*ColumnAttrSet) ProtoMessage() {}
func (*ColumnAttrSet) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{7}
return fileDescriptor_public_5eafd62083455670, []int{7}
}
func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -510,7 +510,7 @@ func (m *Attr) Reset() { *m = Attr{} }
func (m *Attr) String() string { return proto.CompactTextString(m) }
func (*Attr) ProtoMessage() {}
func (*Attr) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{8}
return fileDescriptor_public_5eafd62083455670, []int{8}
}
func (m *Attr) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -592,7 +592,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} }
func (m *AttrMap) String() string { return proto.CompactTextString(m) }
func (*AttrMap) ProtoMessage() {}
func (*AttrMap) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{9}
return fileDescriptor_public_5eafd62083455670, []int{9}
}
func (m *AttrMap) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -644,7 +644,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{10}
return fileDescriptor_public_5eafd62083455670, []int{10}
}
func (m *QueryRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -728,7 +728,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (*QueryResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{11}
return fileDescriptor_public_5eafd62083455670, []int{11}
}
func (m *QueryResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -797,7 +797,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (*QueryResult) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{12}
return fileDescriptor_public_5eafd62083455670, []int{12}
}
func (m *QueryResult) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -907,7 +907,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} }
func (m *ImportRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRequest) ProtoMessage() {}
func (*ImportRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{13}
return fileDescriptor_public_5eafd62083455670, []int{13}
}
func (m *ImportRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1008,7 +1008,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} }
func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) }
func (*ImportValueRequest) ProtoMessage() {}
func (*ImportValueRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_public_fc5da89825239896, []int{14}
return fileDescriptor_public_5eafd62083455670, []int{14}
}
func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1079,6 +1079,116 @@ func (m *ImportValueRequest) GetValues() []int64 {
return nil
}
type ImportRoaringRequestView struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} }
func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) }
func (*ImportRoaringRequestView) ProtoMessage() {}
func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) {
return fileDescriptor_public_5eafd62083455670, []int{15}
}
func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) {
xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src)
}
func (m *ImportRoaringRequestView) XXX_Size() int {
return m.Size()
}
func (m *ImportRoaringRequestView) XXX_DiscardUnknown() {
xxx_messageInfo_ImportRoaringRequestView.DiscardUnknown(m)
}
var xxx_messageInfo_ImportRoaringRequestView proto.InternalMessageInfo
func (m *ImportRoaringRequestView) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *ImportRoaringRequestView) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
type ImportRoaringRequest struct {
Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"`
Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} }
func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRoaringRequest) ProtoMessage() {}
func (*ImportRoaringRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_public_5eafd62083455670, []int{16}
}
func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_ImportRoaringRequest.Merge(dst, src)
}
func (m *ImportRoaringRequest) XXX_Size() int {
return m.Size()
}
func (m *ImportRoaringRequest) XXX_DiscardUnknown() {
xxx_messageInfo_ImportRoaringRequest.DiscardUnknown(m)
}
var xxx_messageInfo_ImportRoaringRequest proto.InternalMessageInfo
func (m *ImportRoaringRequest) GetClear() bool {
if m != nil {
return m.Clear
}
return false
}
func (m *ImportRoaringRequest) GetViews() []*ImportRoaringRequestView {
if m != nil {
return m.Views
}
return nil
}
func init() {
proto.RegisterType((*Row)(nil), "internal.Row")
proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers")
@ -1095,6 +1205,8 @@ func init() {
proto.RegisterType((*QueryResult)(nil), "internal.QueryResult")
proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest")
proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest")
proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView")
proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest")
}
func (m *Row) Marshal() (dAtA []byte, err error) {
size := m.Size()
@ -1979,6 +2091,82 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Name) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Name)))
i += copy(dAtA[i:], m.Name)
}
if len(m.Data) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Data)))
i += copy(dAtA[i:], m.Data)
}
if m.XXX_unrecognized != nil {
i += copy(dAtA[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Clear {
dAtA[i] = 0x8
i++
if m.Clear {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.Views) > 0 {
for _, msg := range m.Views {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.XXX_unrecognized != nil {
i += copy(dAtA[i:], m.XXX_unrecognized)
}
return i, nil
}
func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -2434,6 +2622,47 @@ func (m *ImportValueRequest) Size() (n int) {
return n
}
func (m *ImportRoaringRequestView) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Name)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Data)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *ImportRoaringRequest) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.Clear {
n += 2
}
if len(m.Views) > 0 {
for _, e := range m.Views {
l = e.Size()
n += 1 + l + sovPublic(uint64(l))
}
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovPublic(x uint64) (n int) {
for {
n++
@ -5115,6 +5344,219 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ImportRoaringRequestView: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ImportRoaringRequestView: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Name = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...)
if m.Data == nil {
m.Data = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ImportRoaringRequest: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ImportRoaringRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Clear", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Clear = bool(v != 0)
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Views", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Views = append(m.Views, &ImportRoaringRequestView{})
if err := m.Views[len(m.Views)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipPublic(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
@ -5220,59 +5662,63 @@ var (
ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("public.proto", fileDescriptor_public_fc5da89825239896) }
func init() { proto.RegisterFile("public.proto", fileDescriptor_public_5eafd62083455670) }
var fileDescriptor_public_fc5da89825239896 = []byte{
// 804 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46,
0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28,
0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71,
0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42,
0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e,
0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57,
0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13,
0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4,
0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7,
0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3,
0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c,
0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8,
0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20,
0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0,
0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31,
0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99,
0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94,
0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c,
0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d,
0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc,
0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3,
0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac,
0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28,
0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23,
0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47,
0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32,
0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6,
0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e,
0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d,
0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8,
0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f,
0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9,
0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4,
0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1,
0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf,
0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52,
0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28,
0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a,
0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c,
0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c,
0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99,
0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a,
0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6,
0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0,
0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15,
0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9,
0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47,
0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb,
0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa,
0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25,
0x96, 0x07, 0x00, 0x00,
var fileDescriptor_public_5eafd62083455670 = []byte{
// 870 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x8e, 0xdb, 0x44,
0x14, 0x66, 0x62, 0x27, 0x71, 0x4e, 0x36, 0xa1, 0x1a, 0x2d, 0xc5, 0x42, 0x55, 0xb0, 0x2c, 0x84,
0x7c, 0xb5, 0x95, 0x82, 0x54, 0xf5, 0x8a, 0x9f, 0x6d, 0xb6, 0x28, 0x2a, 0xac, 0xe0, 0x6c, 0x09,
0xe2, 0xd2, 0x6d, 0xa6, 0xad, 0x25, 0xc7, 0x63, 0xec, 0x31, 0x69, 0x9e, 0x83, 0x1b, 0x1e, 0x81,
0x0b, 0x1e, 0xa4, 0x97, 0x88, 0x27, 0x80, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0x64, 0x9c, 0x6c, 0x59,
0x71, 0xc1, 0xdd, 0x7c, 0xe7, 0xcc, 0x39, 0xfe, 0xbe, 0x39, 0x3f, 0x09, 0x9c, 0x94, 0xcd, 0xb3,
0x3c, 0x7b, 0x7e, 0x56, 0x56, 0x52, 0x49, 0x1e, 0x64, 0x85, 0x12, 0x55, 0x91, 0xe6, 0xf1, 0x0f,
0xe0, 0xa1, 0xdc, 0xf2, 0x10, 0x86, 0x8f, 0x64, 0xde, 0x6c, 0x8a, 0x3a, 0x64, 0x91, 0x97, 0xf8,
0x68, 0x21, 0xff, 0x08, 0xfa, 0x5f, 0x28, 0x55, 0xd5, 0x61, 0x2f, 0xf2, 0x92, 0xf1, 0x7c, 0x7a,
0x66, 0x43, 0xcf, 0xb4, 0x19, 0x8d, 0x93, 0x73, 0xf0, 0x9f, 0x88, 0x5d, 0x1d, 0x7a, 0x91, 0x97,
0x8c, 0x90, 0xce, 0xf1, 0x43, 0x98, 0xa2, 0xdc, 0x2e, 0xd7, 0xa2, 0x50, 0xd9, 0x8b, 0x4c, 0x98,
0x5b, 0x28, 0xb7, 0xf6, 0x13, 0x74, 0xde, 0x47, 0xf6, 0x3a, 0x91, 0x9f, 0x82, 0xff, 0x4d, 0x9a,
0x55, 0x7c, 0x0a, 0xbd, 0xe5, 0x22, 0x64, 0x11, 0x4b, 0x7c, 0xec, 0x2d, 0x17, 0xfc, 0x14, 0xfa,
0x8f, 0x64, 0x53, 0xa8, 0xb0, 0x47, 0x26, 0x03, 0xf8, 0x1d, 0xf0, 0x9e, 0x88, 0x5d, 0xe8, 0x45,
0x2c, 0x19, 0xa1, 0x3e, 0xc6, 0x0f, 0x20, 0x78, 0x9c, 0x89, 0x7c, 0xad, 0x95, 0x9d, 0x42, 0x9f,
0xce, 0x94, 0x66, 0x84, 0x06, 0x68, 0xab, 0xe6, 0xb6, 0xb0, 0x99, 0x08, 0xc4, 0x5f, 0x01, 0x7c,
0x59, 0xc9, 0xa6, 0x34, 0x79, 0x13, 0xe8, 0x13, 0x22, 0xba, 0xe3, 0x39, 0x77, 0xca, 0x6d, 0x72,
0x34, 0x17, 0xde, 0xce, 0x2b, 0x9e, 0x43, 0xb0, 0x4a, 0xf3, 0x3d, 0xc7, 0x55, 0x9a, 0x13, 0x07,
0x0f, 0xf5, 0xf1, 0x30, 0xc6, 0xb3, 0x31, 0xdf, 0x81, 0x77, 0x9e, 0x29, 0x47, 0x8f, 0x75, 0xe8,
0xf1, 0x0f, 0x20, 0x30, 0x55, 0xd9, 0xf3, 0xde, 0x63, 0x7e, 0x0f, 0x46, 0x4f, 0xb3, 0x8d, 0xa8,
0x55, 0xba, 0x29, 0xe9, 0x29, 0x3c, 0x74, 0x86, 0xf8, 0x7b, 0x98, 0x98, 0x9b, 0xba, 0x5a, 0x57,
0x42, 0xdd, 0x78, 0xd9, 0xff, 0x56, 0xe5, 0x9b, 0x2f, 0xfd, 0x2b, 0x03, 0x5f, 0xfb, 0xac, 0x8b,
0xed, 0x5d, 0xba, 0xb0, 0x4f, 0x77, 0xa5, 0x68, 0x99, 0xd2, 0x99, 0x47, 0x30, 0xbe, 0x52, 0x55,
0x56, 0xbc, 0x5c, 0xa5, 0x79, 0x23, 0xda, 0x44, 0x5d, 0x93, 0xd6, 0xb8, 0x2c, 0x94, 0x71, 0xfb,
0x24, 0x63, 0x8f, 0xb5, 0xc6, 0x73, 0x29, 0x73, 0xe3, 0xec, 0x47, 0x2c, 0x09, 0xd0, 0x19, 0xf8,
0x0c, 0xe0, 0x71, 0x2e, 0xd3, 0x36, 0x76, 0x10, 0xb1, 0x84, 0x61, 0xc7, 0x12, 0xdf, 0x87, 0xa1,
0x66, 0xfa, 0x75, 0x5a, 0x3a, 0xb5, 0xec, 0x16, 0xb5, 0xf1, 0x1b, 0x06, 0x27, 0xdf, 0x36, 0xa2,
0xda, 0xa1, 0xf8, 0xb1, 0x11, 0x35, 0x55, 0x85, 0xb0, 0x6d, 0x25, 0x02, 0xfc, 0x2e, 0x0c, 0xae,
0x5e, 0xa5, 0xd5, 0xda, 0xbc, 0x9d, 0x8f, 0x2d, 0xd2, 0x5a, 0xdd, 0x9b, 0xd7, 0xa4, 0x35, 0xc0,
0xae, 0x49, 0x47, 0xa2, 0xd8, 0x48, 0x65, 0xc5, 0xb4, 0x88, 0x27, 0xf0, 0xee, 0xc5, 0xeb, 0xe7,
0x79, 0xb3, 0x16, 0x28, 0xb7, 0x26, 0x7a, 0x40, 0x17, 0x8e, 0xcd, 0xfc, 0x63, 0x98, 0xb6, 0x26,
0x3b, 0xbd, 0x43, 0xba, 0x78, 0x64, 0x8d, 0x7f, 0x66, 0x30, 0x69, 0xa5, 0xd4, 0xa5, 0x2c, 0x6a,
0xa1, 0xeb, 0x75, 0x51, 0x55, 0xb6, 0x5e, 0x17, 0x55, 0xc5, 0xef, 0xc3, 0x10, 0x45, 0xdd, 0xe4,
0xca, 0x36, 0xc1, 0x7b, 0xee, 0x59, 0x6c, 0x6c, 0x93, 0x2b, 0xb4, 0xb7, 0xf8, 0x67, 0x30, 0x3d,
0x68, 0x2a, 0x33, 0xfd, 0xe3, 0xf9, 0xfb, 0x2e, 0xee, 0xc0, 0x8f, 0x47, 0xd7, 0xe3, 0x3f, 0x7a,
0x30, 0xee, 0x64, 0xe6, 0x1f, 0xd2, 0x2e, 0x22, 0x4e, 0xe3, 0xf9, 0xc4, 0x65, 0xd1, 0x93, 0x46,
0x5b, 0xea, 0x04, 0xd8, 0x65, 0xdb, 0x4f, 0xec, 0x52, 0x57, 0x51, 0x6f, 0x09, 0xfb, 0xd9, 0x4e,
0x15, 0xb5, 0x19, 0x8d, 0x93, 0x36, 0xdb, 0xab, 0xb4, 0x78, 0x29, 0xd6, 0xd4, 0x4f, 0x01, 0x5a,
0xc8, 0xcf, 0xdc, 0x7c, 0x52, 0x01, 0x0e, 0x46, 0xdc, 0x7a, 0xd0, 0xcd, 0xb0, 0x6d, 0x68, 0x5d,
0x8b, 0x49, 0xdb, 0xd0, 0xba, 0x84, 0x7a, 0x36, 0xf5, 0xc3, 0x53, 0xf1, 0x0d, 0xe2, 0x0f, 0x60,
0xec, 0x36, 0x49, 0x1d, 0x06, 0xc4, 0xf0, 0xd4, 0xa5, 0x77, 0x4e, 0xec, 0x5e, 0xe4, 0x9f, 0x1f,
0xef, 0xcc, 0x70, 0x44, 0xcc, 0xc2, 0x83, 0xd7, 0xe8, 0xf8, 0xf1, 0xe8, 0x7e, 0xfc, 0x17, 0x83,
0xc9, 0x72, 0x53, 0xca, 0x4a, 0x75, 0xda, 0x76, 0x59, 0xac, 0xc5, 0x6b, 0xdb, 0xb6, 0x04, 0xdc,
0x5e, 0xec, 0x1d, 0xed, 0x45, 0x6a, 0x5f, 0x6a, 0x57, 0x1f, 0x0d, 0xe8, 0xa8, 0xf4, 0x0f, 0x54,
0xde, 0x83, 0x91, 0x5d, 0x40, 0x75, 0xd8, 0x27, 0x97, 0x33, 0xe8, 0x81, 0xdc, 0x6f, 0x20, 0xdd,
0xc1, 0x5e, 0xe2, 0x61, 0xc7, 0xa2, 0x2b, 0x83, 0x72, 0x4b, 0xcb, 0x7f, 0x48, 0xcb, 0xdf, 0x42,
0x1d, 0x69, 0xd2, 0x90, 0x33, 0x20, 0x67, 0xc7, 0x12, 0xff, 0xc6, 0x80, 0x1b, 0x8d, 0x34, 0xda,
0xff, 0x9f, 0xd0, 0xdb, 0x05, 0xdd, 0x85, 0x01, 0x7d, 0xcf, 0x8a, 0x69, 0xd1, 0x11, 0xdd, 0xe1,
0x0d, 0xba, 0xe7, 0x10, 0xb6, 0x15, 0x91, 0xa9, 0xde, 0x74, 0x2d, 0xdf, 0x55, 0x26, 0xb6, 0xba,
0xa9, 0x2e, 0xd3, 0x8d, 0x68, 0x29, 0xd3, 0x59, 0xdb, 0x16, 0xa9, 0x4a, 0x89, 0xf0, 0x09, 0xd2,
0x39, 0x7e, 0x01, 0xa7, 0x6f, 0xcb, 0x41, 0x3f, 0x23, 0xb9, 0x48, 0xcd, 0x24, 0x07, 0x68, 0x00,
0x7f, 0x08, 0xfd, 0x9f, 0x32, 0xb1, 0xb5, 0x93, 0x1c, 0xbb, 0xee, 0xf9, 0x37, 0x22, 0x68, 0x02,
0xce, 0xef, 0xbc, 0xb9, 0x9e, 0xb1, 0xdf, 0xaf, 0x67, 0xec, 0xcf, 0xeb, 0x19, 0xfb, 0xe5, 0xef,
0xd9, 0x3b, 0xcf, 0x06, 0xf4, 0x97, 0xe1, 0x93, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, 0x60,
0x42, 0x9e, 0x42, 0x08, 0x00, 0x00,
}

10
internal/public.proto
View file

@ -105,3 +105,13 @@ message ImportValueRequest {
repeated string ColumnKeys = 7;
repeated int64 Values = 6;
}
message ImportRoaringRequestView {
string Name = 1;
bytes Data = 2;
}
message ImportRoaringRequest {
bool Clear = 1;
repeated ImportRoaringRequestView views = 2;
}

22
pilosa_internal_test.go
View file

@ -42,3 +42,25 @@ func TestValidateNameInvalid(t *testing.T) {
}
}
}
// memAttrStore represents an in-memory implementation of the AttrStore interface.
type memAttrStore struct {
store map[uint64]map[string]interface{}
}
func (s *memAttrStore) Path() string { return "" }
func (s *memAttrStore) Open() error { return nil }
func (s *memAttrStore) Close() error { return nil }
func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil }
func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
s.store[id] = m
return nil
}
func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
for id, v := range m {
s.store[id] = v
}
return nil
}
func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }

11
server.go
View file

@ -418,6 +418,9 @@ func (s *Server) loadNodeID() string {
return nodeID
}
// NodeID returns the server's node id.
func (s *Server) NodeID() string { return s.nodeID }
// SyncData manually invokes the anti entropy process which makes sure that this
// node has the data from all replicas across the cluster.
func (s *Server) SyncData() error {
@ -585,7 +588,6 @@ func (s *Server) SendSync(m Message) error {
msg = append([]byte{getMessageType(m)}, msg...)
for _, node := range s.cluster.nodes {
node := node
s.logger.Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if s.uri == node.URI {
continue
@ -606,7 +608,6 @@ func (s *Server) SendAsync(m Message) error {
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, m Message) error {
s.logger.Printf("SendTo: %s", to.URI)
msg, err := s.serializer.Marshal(m)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
@ -630,7 +631,7 @@ func (s *Server) handleRemoteStatus(pb Message) {
go func() {
// Make sure the holder has opened.
<-s.holder.opened
s.holder.opened.Recv()
err := s.mergeRemoteStatus(pb.(*NodeStatus))
if err != nil {
@ -658,7 +659,7 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
// if we don't know about a field locally, log an error because
// fields should be created and synced prior to shard creation
if f == nil {
s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name)
s.logger.Printf("local field not found: %s/%s", is.Name, fs.Name)
continue
}
if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil {
@ -703,7 +704,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.CheckVersion()
err = s.diagnostics.Flush()
if err != nil {
s.logger.Printf("Diagnostics error: %s", err)
s.logger.Printf("diagnostics error: %s", err)
}
}

15
server/config.go
View file

@ -19,6 +19,7 @@ import (
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/toml"
"github.com/uber/jaeger-client-go"
)
// TLSConfig contains TLS configuration
@ -91,6 +92,16 @@ type Config struct {
// Pilosa's developers.
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
Tracing struct {
// SamplerType is the type of sampler to use.
SamplerType string `toml:"sampler-type"`
// SamplerParam is the parameter passed to the tracing sampler.
// Its meaning is dependent on the type of sampler.
SamplerParam float64 `toml:"sampler-param"`
// AgentHostPort is the host:port of the local agent.
AgentHostPort string `toml:"agent-host-port"`
} `toml:"tracing"`
}
// NewConfig returns an instance of Config with default options.
@ -133,5 +144,9 @@ func NewConfig() *Config {
c.Metric.PollInterval = toml.Duration(0 * time.Minute)
c.Metric.Diagnostics = true
// Tracing config.
c.Tracing.SamplerType = jaeger.SamplerTypeRemote
c.Tracing.SamplerParam = 0.001
return c
}

40
server/handler_test.go
View file

@ -22,15 +22,15 @@ import (
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
gohttp "net/http"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
@ -91,9 +91,21 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("ImportRoaring", func(t *testing.T) {
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
req := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(roaringData))
req.Header.Set("Content-Type", "application/x-binary")
h.ServeHTTP(w, req)
msg := pilosa.ImportRoaringRequest{
Clear: false,
Views: map[string][]byte{
"": roaringData,
},
}
ser := proto.Serializer{}
data, err := ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, httpReq)
resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"})
if err != nil {
t.Fatalf("querying: %v", err)
@ -111,9 +123,21 @@ func TestHandler_Endpoints(t *testing.T) {
}
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData))
req.Header.Set("Content-Type", "application/x-binary")
h.ServeHTTP(w, req)
msg := pilosa.ImportRoaringRequest{
Clear: false,
Views: map[string][]byte{
"": roaringData,
},
}
ser := proto.Serializer{}
data, err := ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, httpReq)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}

17
server/server.go
View file

@ -142,7 +142,7 @@ func (m *Command) Start() (err error) {
go func() {
err := m.Handler.Serve()
if err != nil {
m.logger.Printf("Handler serve error: %v", err)
m.logger.Printf("handler serve error: %v", err)
}
}()
@ -151,7 +151,7 @@ func (m *Command) Start() (err error) {
return errors.Wrap(err, "opening server")
}
m.logger.Printf("Listening as %s\n", m.API.Node().URI)
m.logger.Printf("listening as %s\n", m.API.Node().URI)
return nil
}
@ -163,27 +163,31 @@ func (m *Command) Wait() error {
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
select {
case sig := <-c:
m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
m.logger.Printf("received signal '%s', gracefully shutting down...\n", sig.String())
// Second signal causes a hard shutdown.
go func() { <-c; os.Exit(1) }()
return errors.Wrap(m.Close(), "closing command")
case <-m.done:
m.logger.Printf("Server closed externally")
m.logger.Printf("server closed externally")
return nil
}
}
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
var err error
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
m.logOutput = f
err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
if err != nil {
return errors.Wrap(err, "dup2ing stderr onto logfile")
}
}
if m.Config.Verbose {
@ -372,6 +376,7 @@ func (m *Command) Close() error {
eg.Go(closer.Close)
}
}
err := eg.Wait()
return errors.Wrap(err, "closing everything")
}

52
server/server_test.go
View file

@ -621,6 +621,58 @@ func TestMain_ImportTimestamp(t *testing.T) {
}
}
func TestMain_ImportTimestampNoStandardView(t *testing.T) {
m := test.MustRunCommand()
defer m.Close()
indexName := "i"
fieldName := "f-no-standard"
// Create index.
if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
// Create field.
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)); err != nil {
t.Fatal(err)
}
data := pilosa.ImportRequest{
Index: indexName,
Field: fieldName,
Shard: 0,
RowIDs: []uint64{1, 2},
ColumnIDs: []uint64{1, 2},
Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00
}
// Import data.
if err := m.API.Import(context.Background(), &data); err != nil {
t.Fatal(err)
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
exp := []string{
"standard_2018", "standard_201801", "standard_20180101",
"standard_2019", "standard_201912", "standard_20191231",
}
got := []string{}
for _, f := range files {
got = append(got, f.Name())
}
if !reflect.DeepEqual(got, exp) {
t.Fatalf("expected %v, but got %v", exp, got)
}
}
func TestClusterQueriesAfterRestart(t *testing.T) {
cluster := test.MustRunCluster(t, 3)
defer cluster.Close()

60
tracing/opentracing/opentracing.go Normal file
View file

@ -0,0 +1,60 @@
package opentracing
import (
"context"
"log"
"net/http"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/pilosa/pilosa/tracing"
)
// Ensure type implements interface.
var _ tracing.Tracer = (*Tracer)(nil)
// Tracer represents a wrapper for OpenTracing that implements tracing.Tracer.
type Tracer struct {
tracer opentracing.Tracer
}
// NewTracer returns a new instance of Tracer.
func NewTracer(tracer opentracing.Tracer) *Tracer {
return &Tracer{tracer: tracer}
}
// StartSpanFromContext returns a new child span and context from a given context.
func (t *Tracer) StartSpanFromContext(ctx context.Context, operationName string) (tracing.Span, context.Context) {
var opts []opentracing.StartSpanOption
if parent := opentracing.SpanFromContext(ctx); parent != nil {
opts = append(opts, opentracing.ChildOf(parent.Context()))
}
span := t.tracer.StartSpan(operationName, opts...)
return span, opentracing.ContextWithSpan(ctx, span)
}
// InjectHTTPHeaders adds the required HTTP headers to pass context between nodes.
func (t *Tracer) InjectHTTPHeaders(r *http.Request) {
if span := opentracing.SpanFromContext(r.Context()); span != nil {
if err := t.tracer.Inject(
span.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(r.Header),
); err != nil {
log.Printf("opentracing inject error: %s", err)
}
}
}
// ExtractHTTPHeaders reads the HTTP headers to derive incoming context.
func (t *Tracer) ExtractHTTPHeaders(r *http.Request) (tracing.Span, context.Context) {
// Deserialize tracing context into request.
wireContext, _ := t.tracer.Extract(
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(r.Header),
)
span := t.tracer.StartSpan("HTTP", ext.RPCServerOption(wireContext))
ctx := opentracing.ContextWithSpan(r.Context(), span)
return span, ctx
}

58
tracing/tracing.go Normal file
View file

@ -0,0 +1,58 @@
package tracing
import (
"context"
"net/http"
)
// GlobalTracer is a single, global instance of Tracer.
var GlobalTracer Tracer = NopTracer()
// StartSpanFromContext returnus a new child span and context from a given
// context using the global tracer.
func StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context) {
return GlobalTracer.StartSpanFromContext(ctx, operationName)
}
// Tracer implements a generic distributed tracing interface.
type Tracer interface {
// Returns a new child span and context from a given context.
StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context)
// Adds the required HTTP headers to pass context between nodes.
InjectHTTPHeaders(r *http.Request)
// Reads the HTTP headers to derive incoming context.
ExtractHTTPHeaders(r *http.Request) (Span, context.Context)
}
// Span represents a single span in a distributed trace.
type Span interface {
// Sets the end timestamp and finalizes Span state.
Finish()
// Adds key/value pairs to the span.
LogKV(alternatingKeyValues ...interface{})
}
// NopTracer returns a tracer that doesn't do anything.
func NopTracer() Tracer {
return &nopTracer{}
}
type nopTracer struct{}
func (t *nopTracer) StartSpanFromContext(ctx context.Context, operationName string) (Span, context.Context) {
return &nopSpan{}, ctx
}
func (t *nopTracer) InjectHTTPHeaders(r *http.Request) {}
func (t *nopTracer) ExtractHTTPHeaders(r *http.Request) (Span, context.Context) {
return &nopSpan{}, r.Context()
}
type nopSpan struct{}
func (s *nopSpan) Finish() {}
func (s *nopSpan) LogKV(alternatingKeyValues ...interface{}) {}

7
translate.go
View file

@ -195,12 +195,11 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error {
}
// Stop translate store replication.
s.logger.Printf("stop monitor replication")
close(s.replicationClosing)
s.repWG.Wait()
// Set the primary node for translate store replication.
s.logger.Printf("set primary translate store to %s", ev.id)
s.logger.Debugf("set primary translate store to %s", ev.id)
s.primaryID = ev.id
if ev.id == "" {
s.PrimaryTranslateStore = nil
@ -209,7 +208,6 @@ func (s *TranslateFile) handlePrimaryStoreEvent(ev primaryStoreEvent) error {
}
// Start translate store replication. Stream from primary, if available.
s.logger.Printf("start monitor replication")
if s.PrimaryTranslateStore != nil {
s.replicationClosing = make(chan struct{})
s.repWG.Add(1)
@ -386,7 +384,6 @@ func (s *TranslateFile) monitorReplication() {
// monitorPrimaryStoreEvents is executed in a separate goroutine and listens for changes
// to the primary store assignment.
func (s *TranslateFile) monitorPrimaryStoreEvents() {
s.logger.Printf("monitor primary store events")
// Keep handling events until the store closes.
for {
select {
@ -404,7 +401,7 @@ func (s *TranslateFile) replicate(ctx context.Context) error {
off := s.size()
// Connect to remote primary.
s.logger.Printf("pilosa: replicating from offset %d", off)
s.logger.Debugf("pilosa: replicating from offset %d", off)
rc, err := s.PrimaryTranslateStore.Reader(ctx, off)
if err != nil {
return err