scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 03:17:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into vendor-btree

Browse source
This commit is contained in:
Cody Soyland 2018-05-10 11:28:13 -05:00
commit 2590b25628
136 changed files with 17260 additions and 7079 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -1 +0,0 @@
.*

13
.travis.yml
View file

@ -1,14 +1,17 @@
language: go
go:
- 1.8
- 1.9
- "1.10" # Use string, as 1.10==1.1 if interpreted as float.
- master
env:
global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
matrix:
- GOARCH=386
- GOARCH=amd64
install:
- make vendor
- make install-dep install-statik vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
@ -19,10 +22,12 @@ before_deploy:
- pip install awscli --user `whoami`
deploy:
- provider: script
script: make prerelease-upload
script: make prerelease-upload GOOS=linux GOARCH=amd64
skip_cleanup: true
on:
branch: master
all_branches: true
go: "1.10"
condition: $GOARCH = amd64
matrix:
allow_failures:
- go: master

129
CHANGELOG.md
View file

@ -5,6 +5,135 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## [v0.9.0] - 2018-05-04
This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions.
*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.*
### Added
- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191))
- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250))
### Changed
- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220))
- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223))
- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216))
- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226))
- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222))
- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219))
- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212))
- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209))
- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207))
### Fixed
- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233))
- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229))
- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228))
### Deprecated
- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205))
### Performance
- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
## [0.8.8] - 2018-02-19
This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions.
### Fixed
- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
## [0.8.7] - 2018-02-12
This version contains 1 contribution from 1 contributors. There are 2 files changed, 84 insertions, and 4 deletions.
### Fixed
- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
## [0.8.6] - 2018-02-09
This version contains 2 contributions from 2 contributors. There are 3 files changed, 171 insertions, and 6 deletions.
### Fixed
- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
## [0.8.5] - 2018-01-18
This version contains 1 contribution from 1 contributor. There is 1 file changed, 1 insertion, and 0 deletions.
### Fixed
- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
## [0.8.4] - 2018-01-10
This version contains 4 contributions from 3 contributors. There are 17 files changed, 974 insertions, and 221 deletions.
### Fixed
- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
### Performance
- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
## [0.8.3] - 2017-12-12
This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions.

149
CONTRIBUTING.md
View file

@ -2,30 +2,155 @@
## Reporting a bug
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
## Submitting a feature request
Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4]
Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
which starts by [opening a new issue][1] that describes the new feature proposal.
## Submitting code changes
## Making code contributions
Before you start working on new features, you should [open a new issue][1] to let others know what
you're doing before you start working, otherwise you run the risk of duplicating effort. This also
you're doing, otherwise you run the risk of duplicating effort. This also
gives others an opportunity to provide input for your feature.
If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
- Fork the [Pilosa repository][2] and then clone your fork:
```shell
git clone git@github.com:<your-name>/pilosa.git
### Development Environment
- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information.
- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
- Fork the [Pilosa repository][2] to your own account.
- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone git@github.com:${USER}/pilosa.git
```
- `cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
- [Install](https://github.com/golang/dep/#installation) `dep` to manage dependencies:
```sh
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
```
- Install Pilosa command line tools:
```sh
make install
```
or
```
dep ensure && go install github.com/pilosa/pilosa/cmd/...
```
Running `pilosa` should now run a Pilosa instance.
- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add upstream git@github.com:pilosa/pilosa.git
```
### Makefile
Pilosa includes a Makefile that automates several tasks:
- Install Pilosa:
```sh
make install
```
- Install build dependencies (dep, statik, and protoc):
```sh
make install-build-deps
```
- Create the vendor directory:
```sh
make vendor
```
- Run the test suite:
```sh
make test
```
- View the coverage report:
```sh
make cover-viz
```
- Clear the `vendor/` and `build/` directories:
```sh
make clean
```
- Create release tarballs:
```sh
make release
```
- Generate static assets for the WebUI:
```sh
make generate-statik
```
- Regenerate protocol buffer files in `internal/`:
```sh
make generate-protoc
```
- Create tagged Docker image:
```sh
make docker
```
- Run tests inside Docker container:
```sh
make docker-test
```
Additional commands are available in the `Makefile`.
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:
```sh
git fetch upstream
git checkout master
git merge upstream/master
```
- Create a local feature branch:
```shell
```sh
git checkout -b something-amazing
```
@ -33,13 +158,13 @@ If you want to help but you aren't sure where to start, check out our [github la
- Make sure that you've written tests for your new feature, and then run the tests:
```shell
```sh
make test
```
- Verify that your pull request is applied to the latest version of code on github:
```shell
```sh
git remote add upstream git@github.com:pilosa/pilosa.git
git fetch upstream
git rebase -i upstream/master
@ -47,7 +172,7 @@ If you want to help but you aren't sure where to start, check out our [github la
- Push to your fork:
```shell
```sh
git push -u <yourfork> something-amazing
```
@ -59,4 +184,4 @@ If you want to help but you aren't sure where to start, check out our [github la
[3]: https://github.com/pilosa/pilosa/compare/
[4]: https://github.com/pilosa/general/blob/master/proposal.md
[5]: https://github.com/pilosa/pilosa/issues
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

9
Dockerfile
View file

@ -1,12 +1,9 @@
FROM golang:1.9.2 as builder
FROM golang:1.10.2 as builder
ARG ldflags=''
COPY . /go/src/github.com/pilosa/pilosa
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& make vendor \
&& CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa
&& CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
FROM scratch

74
Gopkg.lock generated
View file

@ -37,6 +37,12 @@
revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
version = "v1.3.1"
[[projects]]
name = "github.com/cespare/xxhash"
packages = ["."]
revision = "5c37fe3735342a2e0d01c87a907579987c8936cc"
version = "v1.0.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
@ -51,7 +57,10 @@
[[projects]]
name = "github.com/go-ole/go-ole"
packages = [".","oleutil"]
packages = [
".",
"oleutil"
]
revision = "0e87ea779d9deb219633b828a023b32e1244dd57"
version = "v1.2.0"
@ -61,12 +70,6 @@
revision = "100ba4e885062801d56799d78530b73b178a78f3"
version = "v0.4"
[[projects]]
branch = "master"
name = "github.com/golang/groupcache"
packages = ["lru"]
revision = "84a468cf14b4376def5d68c722b139b881c450a4"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
@ -82,8 +85,8 @@
[[projects]]
name = "github.com/gorilla/mux"
packages = ["."]
revision = "7f08801859139f86dfafd1c296e2cba9a80d292e"
version = "v1.6.0"
revision = "53c1911da2b537f792e7cafcb446b05ffe33b996"
version = "v1.6.1"
[[projects]]
branch = "master"
@ -124,7 +127,17 @@
[[projects]]
branch = "master"
name = "github.com/hashicorp/hcl"
packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
packages = [
".",
"hcl/ast",
"hcl/parser",
"hcl/scanner",
"hcl/strconv",
"hcl/token",
"json/parser",
"json/scanner",
"json/token"
]
revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
[[projects]]
@ -148,7 +161,10 @@
[[projects]]
branch = "master"
name = "github.com/miekg/dns"
packages = [".","internal/socket"]
packages = [
".",
"internal/socket"
]
revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba"
[[projects]]
@ -189,20 +205,30 @@
[[projects]]
name = "github.com/shirou/gopsutil"
packages = ["host","internal/common","mem","process"]
packages = [
"cpu",
"host",
"internal/common",
"mem",
"net",
"process"
]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
[[projects]]
name = "github.com/sony/gobreaker"
branch = "master"
name = "github.com/shirou/w32"
packages = ["."]
revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36"
version = "0.3.0"
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
packages = [".","mem"]
packages = [
".",
"mem"
]
revision = "5660eeed305fe5f69c8fc6cf899132a459a97064"
[[projects]]
@ -250,13 +276,23 @@
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
packages = [
"unix",
"windows"
]
revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
packages = [
"internal/gen",
"internal/triegen",
"internal/ucd",
"transform",
"unicode/cldr",
"unicode/norm"
]
revision = "88f656faf3f37f690df1a32515b479415e1a6769"
[[projects]]
@ -268,6 +304,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56"
inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7"
solver-name = "gps-cdcl"
solver-version = 1

178
Makefile
View file

@ -1,120 +1,138 @@
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
PROTOC := $(shell command -v protoc 2>/dev/null)
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
STATUS := $(shell git status --porcelain)
IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
CLONE_URL=github.com/pilosa/pilosa
PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
BUILD_TIME=`date -u +%FT%T%z`
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
DOCKER_GOLANG_IMAGE=golang:latest
GO_VERSION=latest
default: test pilosa
# Run tests and compile Pilosa
default: test build
# Remove vendor and build directories
clean:
rm -rf vendor build
$(GOPATH)/bin:
mkdir $(GOPATH)/bin
dep: $(GOPATH)/bin
go get -u github.com/golang/dep/cmd/dep
# Set up vendor directory using `dep`
vendor: Gopkg.toml
ifndef DEP
make dep
endif
$(MAKE) require-dep
dep ensure
touch vendor
Gopkg.lock: dep Gopkg.toml
dep ensure
# Run test suite
test: vendor
go test $(PKGS) $(TESTFLAGS)
go test ./... $(TESTFLAGS)
# Run test suite with coverage enabled
cover: vendor
mkdir -p build/coverage
echo "mode: set" > build/coverage/all.out
for pkg in $(PKGS) ; do \
make cover-pkg PKG=$$pkg ; \
done
cover-pkg:
mkdir -p build/coverage
touch build/coverage/$(subst /,-,$(PKG)).out
go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG)
tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
mkdir -p build
$(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
# Run test suite with coverage enabled and view coverage results in browser
cover-viz: cover
go tool cover -html=build/coverage/all.out
go tool cover -html=build/coverage.out
pilosa: vendor
go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Compile Pilosa
build: vendor
go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# Create a single release build under the build directory
release-build: vendor
ifdef DOCKER_BUILD
make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
else
make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
endif
cp NOTICE LICENSE README.md build/pilosa-$(IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID)
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
release:
ifeq ($(STATUS),"")
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
else
@echo "Will not create release with unclean git status."
endif
# Error out if there are untracked changes in Git
check-clean:
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
$(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
# Create branch-tagged pre-release for client library CI jobs
prerelease-build: vendor
make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa"
cp NOTICE LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH)
tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/
@echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz"
$(MAKE) release-build VERSION_ID=$(BRANCH_ID)
# Create prerelease build for Linux/amd64
prerelease:
make prerelease-build GOOS=linux GOARCH=amd64
$(MAKE) prerelease-build GOOS=linux GOARCH=amd64
# Upload prerelease to S3
prerelease-upload: prerelease
aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read
aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read
# Install Pilosa
install: vendor
go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
.protoc-gen-gofast: vendor
ifndef PROTOC
$(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases")
endif
go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast
generate-protoc: .protoc-gen-gofast
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
generate-statik: statik
# `go generate` statik assets (WebUI)
generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa
generate: generate-protoc generate-statik
statik:
ifndef STATIK
go get github.com/rakyll/statik
endif
# `go generate` all needed packages
generate: generate-protoc generate-statik generate-stringer
# Create Docker image from Dockerfile
docker:
docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) .
@echo "Created image: pilosa:$(VERSION)"
docker build -t "pilosa:$(VERSION)" .
@echo Created docker image: pilosa:$(VERSION)
# Compile Pilosa inside Docker container
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS)
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./...
######################
# Build dependencies #
######################
# Verifies that needed build dependency is installed. Errors out if not installed.
define require
$(if $(shell command -v $1 2>/dev/null),
$(info Verified build dependency "$1" is installed.),
$(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`))
endef
require-dep:
$(call require,dep)
require-statik:
$(call require,statik)
require-protoc-gen-gofast:
$(call require,protoc-gen-gofast)
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc
install-dep:
go get -u github.com/golang/dep/cmd/dep
install-statik:
go get -u github.com/rakyll/statik
install-protoc-gen-gofast:
go get -u github.com/gogo/protobuf/protoc-gen-gofast
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html

17
NOTICE
View file

@ -18,6 +18,23 @@ limitations under the License.
Third-party software licenses
=============================
The file /pilosa/lru/lru.go contains a redistribution of lru
(github.com/golang/groupcache/lru); the license follows:
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
The file /pilosa/roaring/btree.go contains a modified redistribution of b
(https://github.com/cznic/b); the license follows:

69
README-dev.md
View file

@ -1,69 +0,0 @@
Development Environment
=======================
Install Go versions 1.6.2+ or 1.7 for your platform.
Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private.
Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`.
Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone git@github.com:${USER}/pilosa.git
```
`cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
Install `dep` to manage dependencies:
```sh
go get -u github.com/golang/dep/cmd/dep
```
Install Pilosa command line tools:
```sh
make install
# or:
# dep ensure && go install github.com/pilosa/pilosa/cmd/...
```
Running `pilosa` should now run a Pilosa instance.
In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add upstream git@github.com:pilosa/pilosa.git
```
Before starting to work on a task, sync your branch with the upstream:
```sh
git fetch upstream
git checkout master
git merge upstream/master
```
Create a branch for the task:
```sh
git checkout -b a-branch-for-the-task
```
Update the code in the branch, and commit it.
Push it to your own repo:
```sh
git push --set-upstream origin a-branch-for-the-task
```
All left to do is creating a pull request on github.com.

1215
api.go Normal file
View file

File diff suppressed because it is too large Load diff

16
apimethod_string.go Normal file
View file

@ -0,0 +1,16 @@
// Code generated by "stringer -type=apiMethod"; DO NOT EDIT.
package pilosa
import "fmt"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return fmt.Sprintf("apiMethod(%d)", i)
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

511
attr.go
View file

@ -16,21 +16,12 @@ package pilosa
import (
"bytes"
"crypto/sha1"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/boltdb/bolt"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// Attribute data type enum.
const (
AttrTypeString = 1
@ -39,313 +30,111 @@ const (
AttrTypeFloat = 4
)
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
// AttrStore represents an interface for handling row/column attributes.
type AttrStore interface {
Path() string
Open() error
Close() error
Attrs(id uint64) (m map[string]interface{}, err error)
SetAttrs(id uint64, m map[string]interface{}) error
SetBulkAttrs(m map[uint64]map[string]interface{}) error
Blocks() ([]AttrBlock, error)
BlockData(i uint64) (map[uint64]map[string]interface{}, error)
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
func init() {
NopAttrStore = &nopAttrStore{}
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
// NopAttrStore represents an AttrStore that doesn't do anything.
var NopAttrStore AttrStore
func NewNopAttrStore(string) AttrStore {
return &nopAttrStore{}
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// nopAttrStore represents a no-op implementation of the AttrStore interface.
type nopAttrStore struct{}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) *AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
// Path is a no-op implementation of AttrStore Path method.
func (s *nopAttrStore) Path() string { return "" }
// Open is a no-op implementation of AttrStore Open method.
func (s *nopAttrStore) Open() error {
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
// Close is a no-op implementation of AttrStore Close method.
func (s *nopAttrStore) Close() error {
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
// Attrs is a no-op implementation of AttrStore Attrs method.
func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return nil, nil
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
return nil
}
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
// Blocks is a no-op implementation of AttrStore Blocks method.
func (s *nopAttrStore) Blocks() ([]AttrBlock, error) {
return nil, nil
}
// BlockData is a no-op implementation of AttrStore BlockData method.
func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []AttrBlock
for cur.nextBlock() {
block := AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := sha1.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
m[btou64(k)] = decodeAttrs(pb.GetAttrs())
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
a, other = a[1:], other[1:]
}
}
// Marshal and save new values.
buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
func encodeAttrs(m map[string]interface{}) []*internal.Attr {
@ -419,136 +208,16 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} {
return other
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
// EncodeAttrs encodes an attribute map into a byte slice.
func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
a, other = a[1:], other[1:]
}
}
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
// DecodeAttrs decodes a byte slice into an attribute map.
func DecodeAttrs(v []byte) (map[string]interface{}, error) {
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}

2
bitmap.go
View file

@ -14,8 +14,6 @@
package pilosa
// #cgo CFLAGS:-mpopcnt
import (
"encoding/json"
"sort"

465
boltdb/attrstore.go Normal file
View file

@ -0,0 +1,465 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []pilosa.AttrBlock
for cur.nextBlock() {
block := pilosa.AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
return nil, err
}
m[btou64(k)] = attrs
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
return pilosa.DecodeAttrs(v)
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
}
}
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}

130
broadcast.go
View file

@ -22,38 +22,27 @@ import (
"github.com/pilosa/pilosa/internal"
)
// NodeSet represents an interface for Node membership and inter-node communication.
type NodeSet interface {
// Returns a list of all Nodes in the cluster
Nodes() []*Node
// Open starts any network activity implemented by the NodeSet
Open() error
// MemberSet represents an interface for Node membership and inter-node communication.
type MemberSet interface {
// Open starts any network activity implemented by the MemberSet
// Node is the local node, used for membership broadcasts.
Open(n *Node) error
}
// StaticNodeSet represents a basic NodeSet for testing.
type StaticNodeSet struct {
// StaticMemberSet represents a basic MemberSet for testing.
type StaticMemberSet struct {
nodes []*Node
}
// NewStaticNodeSet creates a statically defined NodeSet.
func NewStaticNodeSet() *StaticNodeSet {
return &StaticNodeSet{}
// NewStaticMemberSet creates a statically defined MemberSet.
func NewStaticMemberSet(nodes []*Node) *StaticMemberSet {
return &StaticMemberSet{
nodes: nodes,
}
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (s *StaticNodeSet) Nodes() []*Node {
return s.nodes
}
// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing.
func (s *StaticNodeSet) Open() error {
return nil
}
// Join sets the NodeSet nodes to the slice of Nodes passed in.
func (s *StaticNodeSet) Join(nodes []*Node) error {
s.nodes = nodes
// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing.
func (s *StaticMemberSet) Open(n *Node) error {
return nil
}
@ -61,6 +50,7 @@ func (s *StaticNodeSet) Join(nodes []*Node) error {
type Broadcaster interface {
SendSync(pb proto.Message) error
SendAsync(pb proto.Message) error
SendTo(to *Node, pb proto.Message) error
}
func init() {
@ -73,16 +63,21 @@ var NopBroadcaster Broadcaster
type nopBroadcaster struct{}
// SendSync A no-op implemenetation of Broadcaster SendSync method.
// SendSync A no-op implementation of Broadcaster SendSync method.
func (n *nopBroadcaster) SendSync(pb proto.Message) error {
return nil
}
// SendAsync A no-op implemenetation of Broadcaster SendAsync method.
// SendAsync A no-op implementation of Broadcaster SendAsync method.
func (n *nopBroadcaster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error {
return nil
}
// BroadcastHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type BroadcastHandler interface {
@ -117,21 +112,32 @@ var NopGossiper Gossiper
type nopGossiper struct{}
// SendAsync A no-op implemenetation of Gossiper SendAsync method.
// SendAsync A no-op implementation of Gossiper SendAsync method.
func (n *nopGossiper) SendAsync(pb proto.Message) error {
return nil
}
// Broadcast message types.
const (
MessageTypeCreateSlice = 1
MessageTypeCreateIndex = 2
MessageTypeDeleteIndex = 3
MessageTypeCreateFrame = 4
MessageTypeDeleteFrame = 5
MessageTypeCreateInputDefinition = 6
MessageTypeDeleteInputDefinition = 7
MessageTypeDeleteView = 8
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateFrame
MessageTypeDeleteFrame
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateInputDefinition
MessageTypeDeleteInputDefinition
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -148,12 +154,34 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
typ = MessageTypeDeleteFrame
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
case *internal.CreateInputDefinitionMessage:
typ = MessageTypeCreateInputDefinition
case *internal.DeleteInputDefinitionMessage:
typ = MessageTypeDeleteInputDefinition
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -180,12 +208,34 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:
m = &internal.DeleteFrameMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateInputDefinition:
m = &internal.CreateInputDefinitionMessage{}
case MessageTypeDeleteInputDefinition:
m = &internal.DeleteInputDefinitionMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
}

18
broadcast_test.go
View file

@ -15,12 +15,16 @@
package pilosa_test
import (
"bytes"
"reflect"
"testing"
"io/ioutil"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
)
// Ensure a message can be marshaled and unmarshaled.
@ -52,8 +56,18 @@ func testMessageMarshal(t *testing.T, m proto.Message) {
// Ensure that BroadcastReceiver can register a BroadcastHandler.
func TestBroadcast_BroadcastReceiver(t *testing.T) {
s := pilosa.NewServer()
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard)
com.Config.Bind = "localhost:0"
com.Config.DataDir = path
err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO
if err != nil {
t.Fatalf("setting up server: %v", err)
}
s := com.Server
sbr := NewSimpleBroadcastReceiver()
sbh := NewSimpleBroadcastHandler()

2
cache.go
View file

@ -22,8 +22,8 @@ import (
"sync"
"time"
"github.com/golang/groupcache/lru"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/lru"
)
const (

217
client.go
View file

@ -19,7 +19,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -35,6 +34,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
// ClientOptions represents the configuration for a InternalHTTPClient
@ -58,7 +58,7 @@ func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTT
uri, err := NewURIFromAddress(host)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting URI")
}
client := NewInternalHTTPClientFromURI(uri, remoteClient)
@ -88,15 +88,12 @@ func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[st
// maxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.clientURI(ctx), "/slices/max")
u.RawQuery = (&url.Values{
"inverse": {strconv.FormatBool(inverse)},
}).Encode()
u := uriPathToURL(c.defaultURI, "/slices/max")
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -104,18 +101,21 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
var rsp sliceMaxResponse
var rsp getSlicesMaxResponse
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
return rsp.MaxSlices, nil
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
// Schema returns all index and frame schema information.
@ -126,7 +126,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Build request.
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -134,7 +134,7 @@ func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -154,14 +154,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt
Options: opt,
})
if err != nil {
return err
return errors.Wrap(err, "encoding request")
}
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index))
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
@ -171,14 +171,14 @@ func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
return errors.Wrap(err, "reading")
}
// Handle response based on status code.
@ -201,7 +201,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -209,7 +209,7 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -223,8 +223,13 @@ func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, sl
return a, nil
}
// ExecuteQuery executes query against index on the server.
func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
// Query executes query against the index.
func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
if index == "" {
return nil, ErrIndexRequired
} else if queryRequest.Query == "" {
@ -234,14 +239,14 @@ func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, que
// Encode request object.
buf, err := proto.Marshal(queryRequest)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "marshaling")
}
// Create HTTP request.
u := c.clientURI(ctx).Path(fmt.Sprintf("/index/%s/query", index))
u := uri.Path(fmt.Sprintf("/index/%s/query", index))
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
@ -252,14 +257,14 @@ func (c *InternalHTTPClient) ExecuteQuery(ctx context.Context, index string, que
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return nil, errors.New(string(body))
}
@ -296,7 +301,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
// Import to each node.
for _, node := range nodes {
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -317,13 +322,12 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, b
}
node := &Node{
Scheme: c.defaultURI.Scheme(),
Host: c.defaultURI.HostPort(),
URI: *c.defaultURI,
}
// Import to node.
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
return nil
@ -394,7 +398,7 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b
u := nodePathToURL(node, "/import")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
@ -404,14 +408,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
return errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return errors.New(string(body))
}
@ -448,7 +452,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel
// Import to each node.
for _, node := range nodes {
if err := c.importValueNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -482,7 +486,7 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
u := nodePathToURL(node, "/import-value")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
@ -492,14 +496,14 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
return errors.Wrap(err, "reading")
} else if resp.StatusCode != http.StatusOK {
return errors.New(string(body))
}
@ -536,7 +540,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
return nil
@ -560,7 +564,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Generate HTTP request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Accept", "text/csv")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -568,7 +572,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -579,7 +583,7 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
// Copy body to writer.
if _, err := io.Copy(w, resp.Body); err != nil {
return err
return errors.Wrap(err, "copying")
}
return nil
@ -614,13 +618,13 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[index]; i++ {
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
return err
return errors.Wrap(err, "backing up slice")
}
}
// Close tar file.
if err := tw.Close(); err != nil {
return err
return errors.Wrap(err, "closing")
}
return nil
@ -640,9 +644,9 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer,
// Read entire buffer to determine file size.
data, err := ioutil.ReadAll(r)
if err != nil {
return err
return errors.Wrap(err, "reading")
} else if err := r.Close(); err != nil {
return err
return errors.Wrap(err, "closing")
}
// Write slice file header.
@ -652,12 +656,12 @@ func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer,
Size: int64(len(data)),
ModTime: time.Now(),
}); err != nil {
return err
return errors.Wrap(err, "writing header")
}
// Write buffer to file.
if _, err := tw.Write(data); err != nil {
return fmt.Errorf("write buffer: %s", err)
return errors.Wrap(err, "writing buffer")
}
return nil
@ -688,6 +692,13 @@ func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, frame, view, slice, node)
}
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
@ -700,7 +711,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -708,7 +719,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
// Return error if status is not OK.
@ -717,7 +728,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
return nil, ErrFragmentNotFound
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.Host, resp.StatusCode)
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
return resp.Body, nil
@ -740,7 +751,7 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index
if err == io.EOF {
return nil
} else if err != nil {
return err
return errors.Wrap(err, "opening")
}
// Parse slice from entry name.
@ -752,12 +763,12 @@ func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index
// Read file into buffer.
var buf bytes.Buffer
if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil {
return err
return errors.Wrap(err, "copying")
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
return err
return errors.Wrap(err, "restoring")
}
}
}
@ -783,20 +794,20 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
resp.Body.Close()
// Return error if response not OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.Host, resp.StatusCode)
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
}
@ -814,14 +825,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
Options: opt,
})
if err != nil {
return err
return errors.Wrap(err, "marshaling")
}
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame))
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
@ -831,14 +842,14 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
return errors.Wrap(err, "reading")
}
// Handle response based on status code.
@ -862,7 +873,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram
// Build request.
req, err := http.NewRequest("POST", u.String(), nil)
if err != nil {
return err
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -870,7 +881,7 @@ func (c *InternalHTTPClient) RestoreFrame(ctx context.Context, host, index, fram
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
return errors.Wrap(err, "executing request")
}
resp.Body.Close()
@ -888,7 +899,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/views", index, frame))
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -896,7 +907,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -913,7 +924,7 @@ func (c *InternalHTTPClient) FrameViews(ctx context.Context, index, frame string
// Decode response.
var rsp getFrameViewsResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
return nil, errors.Wrap(err, "decoding")
}
return rsp.Views, nil
}
@ -932,7 +943,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -940,7 +951,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -956,7 +967,7 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame, v
// Decode response object.
var rsp getFragmentBlocksResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
return nil, errors.Wrap(err, "decoding")
}
return rsp.Blocks, nil
}
@ -971,13 +982,13 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
Block: uint64(block),
})
if err != nil {
return nil, nil, err
return nil, nil, errors.Wrap(err, "marshaling")
}
u := uriPathToURL(c.defaultURI, "/fragment/block/data")
req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf))
if err != nil {
return nil, nil, err
return nil, nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/protobuf")
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
@ -986,7 +997,7 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, nil, err
return nil, nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -1002,9 +1013,9 @@ func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame, view s
// Decode response object.
var rsp internal.BlockDataResponse
if body, err := ioutil.ReadAll(resp.Body); err != nil {
return nil, nil, err
return nil, nil, errors.Wrap(err, "reading")
} else if err := proto.Unmarshal(body, &rsp); err != nil {
return nil, nil, err
return nil, nil, errors.Wrap(err, "unmarshalling")
}
return rsp.RowIDs, rsp.ColumnIDs, nil
}
@ -1016,13 +1027,13 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Encode request.
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, err
return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -1030,7 +1041,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -1044,7 +1055,7 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
// Decode response object.
var rsp postIndexAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
@ -1056,13 +1067,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Encode request.
buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, err
return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -1070,7 +1081,7 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
@ -1086,19 +1097,19 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
// Decode response object.
var rsp postFrameAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
// SendMessage posts a message synchronously.
func (c *InternalHTTPClient) SendMessage(ctx context.Context, pb proto.Message) error {
func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error {
msg, err := MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
u := uriPathToURL(ctx.Value("uri").(*URI), "/cluster/message")
u := uriPathToURL(uri, "/cluster/message")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("User-Agent", "pilosa/"+Version)
@ -1126,38 +1137,6 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, pb proto.Message)
return nil
}
func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI {
clientURI := c.defaultURI
if contextURI, ok := ctx.Value("uri").(*URI); ok {
clientURI = contextURI
}
return clientURI
}
func (c *InternalHTTPClient) NodeID(uri *URI) (string, error) {
u := uriPathToURL(uri, "/id")
req, err := http.NewRequest("GET", u.String(), nil)
resp, err := c.HTTPClient.Do(req)
if err != nil {
return "", fmt.Errorf("executing http request: %v", err)
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("reading response body: %v", err)
}
// Return error if status is not OK.
switch resp.StatusCode {
case http.StatusOK: // ok
default:
return "", fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body)
}
return string(body), nil
}
// Bit represents the location of a single bit.
type Bit struct {
RowID uint64
@ -1318,8 +1297,8 @@ func uriPathToURL(uri *URI, path string) url.URL {
func nodePathToURL(node *Node, path string) url.URL {
return url.URL{
Scheme: node.Scheme,
Host: node.Host,
Scheme: node.URI.Scheme(),
Host: node.URI.HostPort(),
Path: path,
}
}
@ -1336,7 +1315,8 @@ type InternalClient interface {
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
ExecuteQuery(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error
ImportK(ctx context.Context, index, frame string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@ -1353,6 +1333,5 @@ type InternalClient interface {
BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, pb proto.Message) error
NodeID(uri *URI) (string, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
}

117
client_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -36,10 +37,9 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
for i := 0; i < numNodes; i++ {
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
server[i].Handler.URI = server[i].HostURI()
server[i].Handler.Cluster = c
server[i].Handler.Cluster.Nodes[i].Host = server[i].Host()
server[i].Handler.Holder = hldr[i].Holder
server[i].Handler.API.Cluster = c
server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.API.Holder = hldr[i].Holder
}
return server, hldr
}
@ -47,7 +47,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
var defaultClient *http.Client
func init() {
defaultClient = pilosa.GetHTTPClient(nil)
defaultClient = server.GetHTTPClient(nil)
}
@ -64,24 +64,21 @@ func TestClient_MultiNode(t *testing.T) {
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[0].Holder
e.Scheme = cluster.Nodes[0].Scheme
e.Host = cluster.Nodes[0].Host
e.Node = cluster.Nodes[0]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[1].Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[2].Holder
e.Scheme = cluster.Nodes[2].Scheme
e.Host = cluster.Nodes[2].Host
e.Node = cluster.Nodes[2]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
@ -89,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host())
owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
if ownNum == num {
@ -152,7 +149,7 @@ func TestClient_MultiNode(t *testing.T) {
Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN),
Remote: false,
}
result, err := client[0].ExecuteQuery(context.Background(), "i", queryRequest)
result, err := client[0].Query(context.Background(), "i", queryRequest)
if err != nil {
t.Fatal(err)
}
@ -170,7 +167,7 @@ func TestClient_MultiNode(t *testing.T) {
hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
result, err = client[0].ExecuteQuery(context.Background(), "i", queryRequest)
result, err = client[0].Query(context.Background(), "i", queryRequest)
if err != nil {
t.Fatal(err)
}
@ -190,11 +187,11 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}
result1, err := client[1].ExecuteQuery(context.Background(), "i", queryRequest)
result1, err := client[1].Query(context.Background(), "i", queryRequest)
if err != nil {
t.Fatal(err)
}
result2, err := client[2].ExecuteQuery(context.Background(), "i", queryRequest)
result2, err := client[2].Query(context.Background(), "i", queryRequest)
if err != nil {
t.Fatal(err)
}
@ -220,10 +217,9 @@ func TestClient_Import(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -271,10 +267,9 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -313,17 +308,16 @@ func TestClient_ImportValue(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
if err != nil {
t.Fatal(err)
}
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -335,14 +329,44 @@ func TestClient_ImportValue(t *testing.T) {
t.Fatal(err)
}
// Verify Sum.
sum, cnt, err := frame.FieldSum(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
// Verify data.
if sum != 50 || cnt != 3 {
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt)
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
}
// Verify Min.
min, cnt, err := frame.FieldMin(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -10 || cnt != 1 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt)
}
// Verify Min with Filter.
filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
if err != nil {
t.Fatal(err)
}
min, cnt, err = frame.FieldMin(filter, fld.Name)
if err != nil {
t.Fatal(err)
}
if min != -100 || cnt != 0 {
t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt)
}
// Verify Max.
max, cnt, err := frame.FieldMax(nil, fld.Name)
if err != nil {
t.Fatal(err)
}
if max != 40 || cnt != 1 {
t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt)
}
}
@ -358,10 +382,9 @@ func TestClient_BackupRestore(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -423,10 +446,10 @@ func TestClient_BackupInverseView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -460,10 +483,9 @@ func TestClient_BackupInvalidView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
@ -489,10 +511,9 @@ func TestClient_FragmentBlocks(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)

1751
cluster.go
View file

File diff suppressed because it is too large Load diff

310
cluster_internal_test.go Normal file
View file

@ -0,0 +1,310 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io/ioutil"
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa/internal"
)
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
c := NewCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
tests := []struct {
idx string
maxSlice uint64
frameViews viewsByFrame
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
},
},
{
idx: "foo",
maxSlice: uint64(3),
frameViews: viewsByFrame{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
},
},
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(name string) *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, name)
if err != nil {
panic(err)
}
return index
}
// Ensure that fragSources creates the correct fragment mapping.
func TestFragSources(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
uri3, err := NewURIFromAddress("host3")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node3 := &Node{ID: "node3", URI: *uri3}
c1 := NewCluster()
c1.ReplicaN = 1
c1.addNodeBasicSorted(node0)
c1.addNodeBasicSorted(node1)
c2 := NewCluster()
c2.ReplicaN = 1
c2.addNodeBasicSorted(node0)
c2.addNodeBasicSorted(node1)
c2.addNodeBasicSorted(node2)
c3 := NewCluster()
c3.ReplicaN = 2
c3.addNodeBasicSorted(node0)
c3.addNodeBasicSorted(node1)
c4 := NewCluster()
c4.ReplicaN = 2
c4.addNodeBasicSorted(node0)
c4.addNodeBasicSorted(node1)
c4.addNodeBasicSorted(node2)
c5 := NewCluster()
c5.ReplicaN = 2
c5.addNodeBasicSorted(node0)
c5.addNodeBasicSorted(node1)
c5.addNodeBasicSorted(node2)
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 1300000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 2600000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 3900000, nil)
if err != nil {
t.Fatal(err)
}
tests := []struct {
from *Cluster
to *Cluster
idx *Index
expected map[string][]*internal.ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{},
"node1": []*internal.ResizeSource{},
"node2": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c4,
to: c3,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c5,
to: c4,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
},
"node2": []*internal.ResizeSource{},
},
err: "",
},
{
from: c2,
to: c4,
idx: idx,
expected: nil,
err: "clusters are the same size",
},
{
from: c1,
to: c5,
idx: idx,
expected: nil,
err: "adding more than one node at a time is not supported",
},
{
from: c5,
to: c1,
idx: idx,
expected: nil,
err: "removing more than one node at a time is not supported",
},
}
for _, test := range tests {
actual, err := (test.from).fragSources(test.to, test.idx)
if test.err != "" {
if !strings.Contains(err.Error(), test.err) {
t.Fatalf("expected error: %s, got: %s", test.err, err.Error())
}
} else {
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
}
// Ensure that fragSources creates the correct fragment mapping.
func TestResizeJob(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
tests := []struct {
existingNodes []*Node
node *Node
action string
expectedIDs map[string]bool
}{
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual.IDs, test.expectedIDs) {
t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs)
}
}
}

500
cluster_test.go
View file

@ -12,38 +12,37 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package pilosa
import (
"bytes"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: test.NewModHasher(),
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) {
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) {
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
@ -51,7 +50,7 @@ func TestCluster_Owners(t *testing.T) {
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := pilosa.NewCluster()
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(index, slice)
@ -84,59 +83,448 @@ func TestHasher(t *testing.T) {
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := pilosa.NewHasher().Hash(tt.key, i+1); got != v {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure that an empty cluster returns a valid (empty) NodeSet
func TestCluster_NodeSetHosts(t *testing.T) {
c := pilosa.Cluster{}
if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) {
t.Fatalf("unexpected slice of hosts: %s", h)
}
}
// Ensure cluster can compare its Nodes and Members
func TestCluster_NodeStates(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
NodeSet: &pilosa.StaticNodeSet{},
}
err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{
&pilosa.Node{Host: "serverA:1000"},
&pilosa.Node{Host: "serverC:1000"},
&pilosa.Node{Host: "serverD:1000"},
})
if err != nil {
t.Fatalf("unexpected gossiper nodes: %s", err)
}
// Verify a DOWN node is reported, and extraneous nodes are ignored
if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{
"serverA:1000": pilosa.NodeStateUp,
"serverB:1000": pilosa.NodeStateDown,
"serverC:1000": pilosa.NodeStateUp,
}) {
t.Fatalf("unexpected node state: %s", spew.Sdump(a))
}
}
// Ensure OwnsSlices can find the actual slice list for node and index
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := test.NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
c := NewTestCluster(5)
slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
InverseEnabled: false,
//CacheType: CacheTypeNone,
Fields: []*Field{
{
Name: "fld0",
Type: FieldTypeInt,
Min: -100,
Max: 100,
},
},
}); err != nil {
t.Fatal(err)
}
tc.SetFieldValue("i", "fields", 1, "fld0", -10)
tc.SetFieldValue("i", "fields", 1, "fld0", 10)
tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Frame := node0.Holder.Frame("i", "f")
node0View := node0Frame.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
node0Frame = node0.Holder.Frame("i", "fields")
node0View = node0Frame.View("field_fld0")
node0Fragment = node0View.Fragment(1)
node0ChecksumFld := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Values
// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
node1Frame = node1.Holder.Frame("i", "fields")
node1View = node1Frame.View("field_fld0")
node1Fragment = node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

3
cmd/import.go
View file

@ -60,10 +60,9 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

1
cmd/root_test.go
View file

@ -181,7 +181,6 @@ func TestRootCommand_Config(t *testing.T) {
bind = "127.0.0.1:10101"
[cluster]
poll-interval = "2m0s"
replicas = 2
partitions = 128
hosts = [

59
cmd/server.go
View file

@ -15,18 +15,11 @@
package cmd
import (
"fmt"
"io"
"log"
"os"
"os/signal"
"runtime/pprof"
"syscall"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/ctl"
"github.com/pilosa/pilosa/server"
)
@ -43,55 +36,13 @@ func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Long: `pilosa server runs Pilosa.
It will load existing data from the configured
directory, and start listening client connections
directory and start listening for client connections
on the configured port.`,
RunE: func(cmd *cobra.Command, args []string) error {
logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr)
if err != nil {
return err
if err := Server.Start(); err != nil {
return errors.Wrap(err, "running server")
}
logger := log.New(logOutput, "", log.LstdFlags)
logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
// Start CPU profiling.
if Server.CPUProfile != "" {
f, err := os.Create(Server.CPUProfile)
if err != nil {
return fmt.Errorf("create cpu profile: %v", err)
}
defer f.Close()
fmt.Fprintln(Server.Stderr, "Starting cpu profile")
pprof.StartCPUProfile(f)
time.AfterFunc(Server.CPUTime, func() {
fmt.Fprintln(Server.Stderr, "Stopping cpu profile")
pprof.StopCPUProfile()
f.Close()
})
}
// Execute the program.
if err := Server.Run(); err != nil {
return fmt.Errorf("error running server: %v", err)
}
// First SIGKILL causes server to shut down gracefully.
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
select {
case sig := <-c:
logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
// Second signal causes a hard shutdown.
go func() { <-c; os.Exit(1) }()
if err := Server.Close(); err != nil {
return err
}
case <-Server.Done:
logger.Printf("Server closed externally")
}
return nil
return errors.Wrap(Server.Wait(), "waiting on Server")
},
}

32
cmd/server_test.go
View file

@ -21,9 +21,9 @@ import (
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/cmd"
_ "github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa/toml"
)
func TestServerHelp(t *testing.T) {
@ -37,23 +37,20 @@ func TestServerHelp(t *testing.T) {
func TestServerConfig(t *testing.T) {
actualDataDir, err := ioutil.TempDir("", "")
failErr(t, err, "making data dir")
profFile, err := ioutil.TempFile("", "")
failErr(t, err, "making temp file")
logFile, err := ioutil.TempFile("", "")
failErr(t, err, "making log file")
tests := []commandTest{
// TEST 0
{
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
bind = "localhost:0"
max-writes-per-request = 3000
[cluster]
poll-interval = "45s"
type = "static"
disabled = true
replicas = 2
hosts = [
"localhost:19444",
@ -66,8 +63,7 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.Bind, "localhost:10111")
v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
return v.Error()
},
@ -80,7 +76,7 @@ func TestServerConfig(t *testing.T) {
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
[cluster]
type = "static"
disabled = true
hosts = [
"localhost:19444",
]
@ -88,27 +84,23 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9))
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
return v.Error()
},
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.type", "static"},
env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"},
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:19444"
data-dir = "` + actualDataDir + `"
[cluster]
poll-interval = "2m0s"
hosts = [
"localhost:19444",
]
[anti-entropy]
interval = "11m0s"
[profile]
cpu = "` + profFile.Name() + `"
cpu-time = "35s"
[metric]
service = "statsd"
host = "127.0.0.1:8125"
@ -116,10 +108,7 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Minute*2))
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
v.Check(cmd.Server.CPUProfile, profFile.Name())
v.Check(cmd.Server.CPUTime, time.Minute)
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
v.Check(cmd.Server.Config.LogPath, logFile.Name())
v.Check(cmd.Server.Config.Metric.Service, "statsd")
v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125")
@ -151,6 +140,9 @@ func TestServerConfig(t *testing.T) {
case <-cmd.Server.Started:
case <-executed:
}
if execErr != nil {
t.Fatalf("executing server command: %v", execErr)
}
err := cmd.Server.Close()
failErr(t, err, "closing pilosa server command")
<-executed

180
config.go
View file

@ -1,180 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"time"
)
// Cluster types.
const (
ClusterNone = ""
ClusterStatic = "static"
ClusterGossip = "gossip"
)
const (
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultPort is the default port use with the hostname.
DefaultPort = "10101"
// DefaultClusterType sets the node intercommunication method.
DefaultClusterType = ClusterGossip
// DefaultGossipPort indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// DefaultMetrics sets the internal metrics to no-op.
DefaultMetrics = "nop"
// DefaultMaxWritesPerRequest is the default number of writes per request.
DefaultMaxWritesPerRequest = 5000
)
// ClusterTypes set of cluster types.
var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterGossip}
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)
CertificatePath string `toml:"certificate-path"`
// CertificateKeyPath contains the path to the certificate key (.key file)
CertificateKeyPath string `toml:"certificate-key-path"`
// SkipVerify disables verification for self-signed certificates
SkipVerify bool `toml:"skip-verify"`
}
// Config represents the configuration for the command.
type Config struct {
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
// GossipPort DEPRECATED
GossipPort string `toml:"gossip-port"`
// GossipSeed DEPRECATED
GossipSeed string `toml:"gossip-seed"`
Gossip struct {
Port string `toml:"port"`
Seed string `toml:"seed"`
Key string `toml:"key"`
} `toml:"gossip"`
Cluster struct {
ReplicaN int `toml:"replicas"`
Type string `toml:"type"`
Hosts []string `toml:"hosts"`
PollInterval Duration `toml:"poll-interval"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
// Limits the number of mutating commands that can be in a single request to
// the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`
LogPath string `toml:"log-path"`
Metric struct {
Service string `toml:"service"`
Host string `toml:"host"`
PollInterval Duration `toml:"poll-interval"`
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
TLS TLSConfig
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
Bind: DefaultHost + ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
}
c.Cluster.ReplicaN = DefaultReplicaN
c.Cluster.Type = DefaultClusterType
c.Cluster.PollInterval = Duration(DefaultPollingInterval)
c.Cluster.Hosts = []string{}
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
c.Metric.Service = DefaultMetrics
c.Metric.Diagnostics = true
c.TLS = TLSConfig{}
return c
}
// Validate that all configuration permutations are compatible with each other.
func (c *Config) Validate() error {
if !StringInSlice(c.Cluster.Type, ClusterTypes) {
return ErrConfigClusterTypeInvalid
}
if c.Cluster.Type == ClusterGossip {
if len(c.Cluster.Hosts) > 0 {
bindWithDefaults, err := AddressWithDefaults(c.Bind)
if err != nil {
return err
}
if !c.foundHost(bindWithDefaults) {
return ErrConfigHostsMissing
}
}
}
return nil
}
func (c *Config) foundHost(host *URI) bool {
for _, clusterHost := range c.Cluster.Hosts {
uri, err := NewURIFromAddress(clusterHost)
if err != nil {
continue
}
if host.Equals(uri) {
return true
}
}
return false
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration
// String returns the string representation of the duration.
func (d Duration) String() string { return time.Duration(d).String() }
// UnmarshalText parses a TOML value into a duration value.
func (d *Duration) UnmarshalText(text []byte) error {
v, err := time.ParseDuration(string(text))
if err != nil {
return err
}
*d = Duration(v)
return nil
}
// MarshalText writes duration value in text format.
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
}
// MarshalTOML write duration into valid TOML.
func (d Duration) MarshalTOML() ([]byte, error) {
return []byte(d.String()), nil
}

17
ctl/backup.go
View file

@ -16,11 +16,12 @@ package ctl
import (
"context"
"errors"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BackupCommand represents a command for backing up a view.
@ -39,7 +40,7 @@ type BackupCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewBackupCommand returns a new instance of BackupCommand.
@ -59,26 +60,26 @@ func (cmd *BackupCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return err
return errors.Wrap(err, "creating client")
}
// Open output file.
f, err := os.Create(cmd.Path)
if err != nil {
return err
return errors.Wrap(err, "creating file")
}
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return err
return errors.Wrap(err, "backing up")
}
// Sync & close file to ensure durability.
if err := f.Sync(); err != nil {
return err
return errors.Wrap(err, "syncing")
} else if err = f.Close(); err != nil {
return err
return errors.Wrap(err, "closing file")
}
return nil
@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

12
ctl/backup_test.go
View file

@ -46,14 +46,10 @@ func TestBackupCommand_Run(t *testing.T) {
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")

11
ctl/bench.go
View file

@ -16,7 +16,6 @@ package ctl
import (
"context"
"errors"
"fmt"
"io"
"math/rand"
@ -24,6 +23,8 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BenchCommand represents a command for benchmarking index operations.
@ -42,7 +43,7 @@ type BenchCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewBenchCommand returns a new instance of BenchCommand.
@ -57,7 +58,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return err
return errors.Wrap(err, "creating client")
}
switch cmd.Op {
@ -94,7 +95,7 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalCl
Query: fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID),
Remote: false,
}
if _, err := client.ExecuteQuery(ctx, cmd.Index, queryRequest); err != nil {
if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil {
return err
}
}
@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

3
ctl/bench_test.go
View file

@ -23,6 +23,7 @@ import (
"testing"
"github.com/pilosa/pilosa"
"github.com/pkg/errors"
)
func TestBenchCommand_InvalidOption(t *testing.T) {
@ -31,7 +32,7 @@ func TestBenchCommand_InvalidOption(t *testing.T) {
cm := NewBenchCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
if err != pilosa.ErrHostRequired {
if errors.Cause(err) != pilosa.ErrHostRequired {
t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err)
}

15
ctl/check.go
View file

@ -24,6 +24,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
"github.com/pkg/errors"
)
// CheckCommand represents a command for performing consistency checks on data files.
@ -48,17 +49,17 @@ func (cmd *CheckCommand) Run(ctx context.Context) error {
switch filepath.Ext(path) {
case "":
if err := cmd.checkBitmapFile(path); err != nil {
return err
return errors.Wrap(err, "checking bitmap")
}
case ".cache":
if err := cmd.checkCacheFile(path); err != nil {
return err
return errors.Wrap(err, "checking cache")
}
case ".snapshotting":
if err := cmd.checkSnapshotFile(path); err != nil {
return err
return errors.Wrap(err, "checking snapshot")
}
}
}
@ -71,26 +72,26 @@ func (cmd *CheckCommand) checkBitmapFile(path string) error {
// Open file handle.
f, err := os.Open(path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return err
return errors.Wrap(err, "statting file")
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return err
return errors.Wrap(err, "mmapping")
}
defer syscall.Munmap(data)
// Attach the mmap file to the bitmap.
bm := roaring.NewSliceBitmap()
if err := bm.UnmarshalBinary(data); err != nil {
return err
return errors.Wrap(err, "unmarshalling")
}
// Perform consistency check.

2
ctl/check_test.go
View file

@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) {
var buf bytes.Buffer
io.Copy(&buf, r)
if !strings.HasPrefix(err.Error(), "invalid roaring file") {
if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") {
t.Fatalf("expect error: invalid roaring file, actual: '%s'", err)
}
// Todo: need correct roaring file for happy path

10
ctl/common.go
View file

@ -18,13 +18,15 @@ import (
"crypto/tls"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
"github.com/spf13/pflag"
)
// CommandWithTLSSupport is the interface for commands which has TLS settings
type CommandWithTLSSupport interface {
TLSHost() string
TLSConfiguration() pilosa.TLSConfig
TLSConfiguration() server.TLSConfig
}
// SetTLSConfig creates common TLS flags
@ -41,16 +43,16 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
cert, err := tls.LoadX509KeyPair(tlsConfig.CertificatePath, tlsConfig.CertificateKeyPath)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "loading keypair")
}
TLSConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: tlsConfig.SkipVerify,
}
}
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig))
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting internal client")
}
return client, err
}

3
ctl/config.go
View file

@ -21,12 +21,13 @@ import (
toml "github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
)
// ConfigCommand represents a command for printing a default config.
type ConfigCommand struct {
*pilosa.CmdIO
Config *pilosa.Config
Config *server.Config
}
// NewConfigCommand returns a new instance of ConfigCommand.

9
ctl/config_test.go
View file

@ -17,11 +17,12 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
"testing"
"github.com/pilosa/pilosa/server"
)
func TestConfigCommand_Run(t *testing.T) {
@ -29,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) {
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewConfigCommand(stdin, w, os.Stderr)
cm.Config = pilosa.NewConfig()
cm.Config = server.NewConfig()
err := cm.Run(context.Background())
w.Close()
@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) {
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
t.Fatalf("Unexpected config: %s", buf.String())
} else if !strings.Contains(buf.String(), ":10101") {
t.Fatalf("Unexpected config: \n%s", buf.String())
}
}

16
ctl/export.go
View file

@ -21,6 +21,8 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// ExportCommand represents a command for bulk exporting data from a server.
@ -38,7 +40,7 @@ type ExportCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewExportCommand returns a new instance of ExportCommand.
@ -67,7 +69,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
if cmd.Path != "" {
f, err := os.Create(cmd.Path)
if err != nil {
return err
return errors.Wrap(err, "creating file")
}
defer f.Close()
@ -77,7 +79,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return err
return errors.Wrap(err, "creating client")
}
// Determine slice count.
@ -89,21 +91,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
if err != nil {
return err
return errors.Wrap(err, "getting slice count")
}
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
return err
return errors.Wrap(err, "exporting")
}
}
// Close writer, if applicable.
if w, ok := w.(io.Closer); ok {
if err := w.Close(); err != nil {
return err
return errors.Wrap(err, "closing")
}
}
@ -114,6 +116,6 @@ func (cmd *ExportCommand) TLSHost() string {
return cmd.Host
}
func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

15
ctl/export_test.go
View file

@ -59,14 +59,10 @@ func TestExportCommand_Run(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
@ -75,8 +71,7 @@ func TestExportCommand_Run(t *testing.T) {
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
err = cm.Run(context.Background())
if err != nil {
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}
}

1
ctl/generate_config.go
View file

@ -43,7 +43,6 @@ bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
poll-interval = "2m0s"
replicas = 1
hosts = [
"localhost:10101",

3
ctl/generate_config_test.go
View file

@ -17,7 +17,6 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
io.Copy(&buf, r)
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
} else if !strings.Contains(buf.String(), "localhost:10101") {
t.Fatalf("Unexpected config: %s", buf.String())
}
}

29
ctl/import.go
View file

@ -17,7 +17,6 @@ package ctl
import (
"context"
"encoding/csv"
"errors"
"fmt"
"io"
"log"
@ -27,6 +26,8 @@ import (
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// ImportCommand represents a command for bulk importing data.
@ -66,7 +67,7 @@ type ImportCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewImportCommand returns a new instance of ImportCommand.
@ -93,14 +94,14 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return err
return errors.Wrap(err, "creating client")
}
cmd.Client = client
if cmd.CreateSchema {
err := cmd.ensureSchema(ctx)
if err != nil {
return err
return errors.Wrap(err, "ensuring schema")
}
}
@ -152,7 +153,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
// Open file for reading.
f, err := os.Open(path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
@ -172,7 +173,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
if err == io.EOF {
break
} else if err != nil {
return err
return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@ -242,7 +243,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
return err
return errors.Wrap(err, "importing")
}
}
@ -259,7 +260,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
// Open file for reading.
f, err := os.Open(path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
@ -279,7 +280,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
if err == io.EOF {
break
} else if err != nil {
return err
return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@ -339,7 +340,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
return err
return errors.Wrap(err, "importing keys")
}
return nil
@ -355,7 +356,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
// Open file for reading.
f, err := os.Open(path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
@ -375,7 +376,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
if err == io.EOF {
break
} else if err != nil {
return err
return errors.Wrap(err, "reading")
}
// Ignore blank rows.
@ -436,7 +437,7 @@ func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.F
logger.Printf("importing slice: %d, n=%d", slice, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil {
return err
return errors.Wrap(err, "importing values")
}
}
@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string {
return cmd.Host
}
func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

34
ctl/import_test.go
View file

@ -65,14 +65,10 @@ func TestImportCommand_Run(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
cm.Index = "i"
@ -85,10 +81,8 @@ func TestImportCommand_Run(t *testing.T) {
}
}
// Ensure that the ImportValue path runs (note: we have specifed a value
// for cm.Field. Because the handler doesn't return errors (it sends them
// to the logger), we don't get an error returned at `cm.Run()` even though
// we haven't setup frame `f` to be RangeEnabled.
// Ensure that the ImportValue path runs (note: we have specified a value
// for cm.Field.)
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
@ -105,18 +99,14 @@ func TestImportCommand_RunValue(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
cm.Index = "i"
cm.Frame = "f"
@ -133,7 +123,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = pilosa.DefaultHost
cm.Host = "anyhost"
cm.Index = "i"
cm.Frame = "f"
file, err := ioutil.TempFile("", "import.csv")

9
ctl/inspect.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
"github.com/pkg/errors"
)
// InspectCommand represents a command for inspecting fragment data files.
@ -49,19 +50,19 @@ func (cmd *InspectCommand) Run(ctx context.Context) error {
// Open file handle.
f, err := os.Open(cmd.Path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return err
return errors.Wrap(err, "statting file")
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return err
return errors.Wrap(err, "mmapping")
}
defer syscall.Munmap(data)
@ -70,7 +71,7 @@ func (cmd *InspectCommand) Run(ctx context.Context) error {
fmt.Fprintf(cmd.Stderr, "unmarshaling bitmap...")
bm := roaring.NewSliceBitmap()
if err := bm.UnmarshalBinary(data); err != nil {
return err
return errors.Wrap(err, "unmarshalling")
}
fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t))

13
ctl/restore.go
View file

@ -16,11 +16,12 @@ package ctl
import (
"context"
"errors"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// RestoreCommand represents a command for restoring a frame from a backup.
@ -39,7 +40,7 @@ type RestoreCommand struct {
// Standard input/output
*pilosa.CmdIO
TLS pilosa.TLSConfig
TLS server.TLSConfig
}
// NewRestoreCommand returns a new instance of RestoreCommand.
@ -59,19 +60,19 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return err
return errors.Wrap(err, "creating client")
}
// Open backup file.
f, err := os.Open(cmd.Path)
if err != nil {
return err
return errors.Wrap(err, "opening file")
}
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return err
return errors.Wrap(err, "restoring")
}
return nil
@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string {
return cmd.Host
}
func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig {
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

12
ctl/restore_test.go
View file

@ -48,14 +48,10 @@ func TestRestoreCommand_Run(t *testing.T) {
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)
cm.Path = file.Name()

55
ctl/server.go
View file

@ -24,26 +24,41 @@ import (
// BuildServerFlags attaches a set of flags to the command for a server instance.
func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this?
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path")
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.")
flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]")
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
// TLS
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
// Cluster
flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
// AntiEntropy
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
// Metric
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
}

3
ctl/server_test.go
View file

@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) {
stdin, stdout, stderr := GetIO(buf)
Server := server.NewCommand(stdin, stdout, stderr)
BuildServerFlags(cm, Server)
if cm.Flags().Lookup("gossip-port").Name == "" {
t.Fatal("gossip-port flag is required")
}
if cm.Flags().Lookup("data-dir").Name == "" {
t.Fatal("data-dir flag is required")
}

309
diagnostics.go Normal file
View file

@ -0,0 +1,309 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"sync"
"time"
)
// Default version check URL.
const (
defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// DiagnosticsCollector represents a collector/sender of diagnostics data.
type DiagnosticsCollector struct {
mu sync.Mutex
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
Logger Logger
server *Server
}
// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
return &DiagnosticsCollector{
host: host,
VersionURL: defaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: &http.Client{Timeout: 10 * time.Second},
metrics: make(map[string]interface{}),
Logger: NopLogger,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *DiagnosticsCollector) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// Flush sends the current metrics.
func (d *DiagnosticsCollector) Flush() error {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, err := d.encode()
if err != nil {
return err
}
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return err
}
// Intentionally ignoring response body, as user does not need to be notified of error.
defer resp.Body.Close()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *DiagnosticsCollector) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// If version has not changed since the last check, return
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.compareVersion(rsp.Version); err != nil {
d.Logger.Printf("%s\n", err.Error())
}
return nil
}
// compareVersion check version strings.
func (d *DiagnosticsCollector) compareVersion(value string) error {
currentVersion := versionSegments(value)
localVersion := versionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *DiagnosticsCollector) encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *DiagnosticsCollector) Set(name string, value interface{}) {
switch v := value.(type) {
case string:
if v == "" {
// Do not set empty string
return
}
}
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// logErr logs the error and returns true if an error exists
func (d *DiagnosticsCollector) logErr(err error) bool {
if err != nil {
d.Logger.Printf("%v", err)
return true
}
return false
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.systemInfo.Uptime()
if !d.logErr(err) {
d.Set("HostUptime", uptime)
}
platform, err := d.server.systemInfo.Platform()
if !d.logErr(err) {
d.Set("OSPlatform", platform)
}
family, err := d.server.systemInfo.Family()
if !d.logErr(err) {
d.Set("OSFamily", family)
}
version, err := d.server.systemInfo.OSVersion()
if !d.logErr(err) {
d.Set("OSVersion", version)
}
kernelVersion, err := d.server.systemInfo.KernelVersion()
if !d.logErr(err) {
d.Set("OSKernelVersion", kernelVersion)
}
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
memFree, err := d.server.systemInfo.MemFree()
if !d.logErr(err) {
d.Set("MemFree", memFree)
}
memTotal, err := d.server.systemInfo.MemTotal()
if !d.logErr(err) {
d.Set("MemTotal", memTotal)
}
memUsed, err := d.server.systemInfo.MemUsed()
if !d.logErr(err) {
d.Set("MemUsed", memUsed)
}
}
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range d.server.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}
// versionSegments returns the numeric segments of the version as a slice of ints.
func versionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}
// SystemInfo collects information about the host OS.
type SystemInfo interface {
Uptime() (uint64, error)
Platform() (string, error)
Family() (string, error)
OSVersion() (string, error)
KernelVersion() (string, error)
MemFree() (uint64, error)
MemTotal() (uint64, error)
MemUsed() (uint64, error)
}
// NewNopSystemInfo creates a no-op implementation of SystemInfo.
func NewNopSystemInfo() *NopSystemInfo {
return &NopSystemInfo{}
}
// NopSystemInfo is a no-op implementation of SystemInfo.
type NopSystemInfo struct {
}
// Uptime is a no-op implementation of SystemInfo.Uptime.
func (n *NopSystemInfo) Uptime() (uint64, error) {
return 0, nil
}
// Platform is a no-op implementation of SystemInfo.Platform.
func (n *NopSystemInfo) Platform() (string, error) {
return "", nil
}
// Family is a no-op implementation of SystemInfo.Family.
func (n *NopSystemInfo) Family() (string, error) {
return "", nil
}
// OSVersion is a no-op implementation of SystemInfo.OSVersion.
func (n *NopSystemInfo) OSVersion() (string, error) {
return "", nil
}
// KernelVersion is a no-op implementation of SystemInfo.KernelVersion.
func (n *NopSystemInfo) KernelVersion() (string, error) {
return "", nil
}
// MemFree is a no-op implementation of SystemInfo.MemFree.
func (n *NopSystemInfo) MemFree() (uint64, error) {
return 0, nil
}
// MemTotal is a no-op implementation of SystemInfo.MemTotal.
func (n *NopSystemInfo) MemTotal() (uint64, error) {
return 0, nil
}
// MemUsed is a no-op implementation of SystemInfo.MemUsed.
func (n *NopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}

267
diagnostics/diagnostics.go
View file

@ -1,267 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"github.com/sony/gobreaker"
)
// TODO: unique Cluster ID
// Default version check URL.
const (
DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// Diagnostics represents a client to the Pilosa cluster.
type Diagnostics struct {
mu sync.Mutex
wg sync.WaitGroup
closing chan struct{}
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
interval time.Duration
cb *gobreaker.CircuitBreaker
logOutput io.Writer
}
// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port".
func New(host string) *Diagnostics {
return &Diagnostics{
closing: make(chan struct{}),
host: host,
VersionURL: DefaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: http.DefaultClient,
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *Diagnostics) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// SetInterval of the diagnostic go routine and match with the circuit breaker timeout.
func (d *Diagnostics) SetInterval(i time.Duration) {
d.interval = i
}
// schedule start the diagnostics service ticker.
func (d *Diagnostics) schedule() {
ticker := time.NewTicker(d.interval)
defer ticker.Stop()
for {
select {
case <-d.closing:
return
case <-ticker.C:
d.CheckVersion()
d.Flush()
}
}
}
// Flush sends the current metrics.
func (d *Diagnostics) Flush() error {
d.mu.Lock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, _ := d.Encode()
d.mu.Unlock()
_, err := d.cb.Execute(func() (interface{}, error) {
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// TODO verify response
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
})
return err
}
// Open configures the circuit breaker used by the HTTP client.
func (d *Diagnostics) Open() {
var st gobreaker.Settings
if d.interval > 0 {
st.Timeout = d.interval * 2
}
d.cb = gobreaker.NewCircuitBreaker(st)
d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics")
}
// Close notify goroutine to stop.
func (d *Diagnostics) Close() error {
close(d.closing)
d.wg.Wait()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *Diagnostics) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// Same a version as last test
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.CompareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
}
return nil
}
// CompareVersion check version strings.
func (d *Diagnostics) CompareVersion(value string) error {
currentVersion := VersionSegments(value)
localVersion := VersionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *Diagnostics) Encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *Diagnostics) Set(name string, value interface{}) {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *Diagnostics) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *Diagnostics) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *Diagnostics) EnrichWithOSInfo() {
osInfo, err := host.Info()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("HostUptime", osInfo.Uptime)
platform, family, version, err := host.PlatformInformation()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSPlatform", platform)
d.Set("OSFamily", family)
d.Set("OSVersion", version)
kernelVersion, err := host.KernelVersion()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSKernelVersion", kernelVersion)
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *Diagnostics) EnrichWithMemoryInfo() {
memory, err := mem.VirtualMemory()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("MemFree", memory.Free)
d.Set("MemTotal", memory.Total)
d.Set("MemUsed", memory.Used)
}
// VersionSegments returns the numeric segments of the version as a slice of ints.
func VersionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}

46
diagnostics/diagnostics_test.go → diagnostics_internal_test.go
View file

@ -12,36 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics_test
package pilosa
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"reflect"
"runtime"
"strings"
"testing"
"github.com/pilosa/pilosa/diagnostics"
)
func TestDiagnosticsClient(t *testing.T) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d.SetLogger(ioutil.Discard)
d.Open()
defer d.Close()
d := NewDiagnosticsCollector(server.URL)
d.Set("gg", 10)
d.Set("ss", "ss")
data, err := d.Encode()
data, err := d.encode()
if err != nil {
t.Fatal(err)
}
@ -58,7 +51,7 @@ func TestDiagnosticsClient(t *testing.T) {
// Test the metrics after a flush.
d.Flush()
data, err = d.Encode()
data, err = d.encode()
if err != nil {
t.Fatal(err)
}
@ -74,7 +67,7 @@ func TestDiagnosticsClient(t *testing.T) {
func TestDiagnosticsVersion_Parse(t *testing.T) {
version := "0.1.1"
vs := diagnostics.VersionSegments(version)
vs := versionSegments(version)
output := []int{0, 1, 1}
if !reflect.DeepEqual(vs, output) {
@ -83,35 +76,33 @@ func TestDiagnosticsVersion_Parse(t *testing.T) {
}
func TestDiagnosticsVersion_Compare(t *testing.T) {
d := diagnostics.New("localhost:10101")
d.Open()
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "v0.1.1"
d.SetVersion(version)
err := d.CompareVersion("v1.7.0")
err := d.compareVersion("v1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("1.7.0")
err = d.compareVersion("1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("0.7.0")
err = d.compareVersion("0.7.0")
if !strings.Contains(err.Error(), "The latest Minor release is") {
t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.2")
err = d.compareVersion("0.1.2")
if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") {
t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.1")
err = d.compareVersion("0.1.1")
if err != nil {
t.Fatalf("Versions should match")
}
d.SetVersion("v1.7.0")
err = d.CompareVersion("0.7.2")
err = d.compareVersion("0.7.2")
if err != nil {
t.Fatalf("Local version is greater")
}
@ -125,11 +116,9 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
Version: "1.1.1",
})
}))
defer server.Close()
// Create a new client.
d := diagnostics.New("localhost:10101")
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "0.1.1"
d.SetVersion(version)
@ -138,10 +127,6 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
d.CheckVersion()
}
type versionResponse struct {
Version string `json:"version"`
}
func compareJSON(a, b []byte) (bool, error) {
var j1, j2 interface{}
if err := json.Unmarshal(a, &j1); err != nil {
@ -156,12 +141,9 @@ func compareJSON(a, b []byte) (bool, error) {
func BenchmarkDiagnostics(b *testing.B) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d.SetLogger(ioutil.Discard)
defer d.Close()
d := NewDiagnosticsCollector(server.URL)
prev := runtime.GOMAXPROCS(4)
defer runtime.GOMAXPROCS(prev)

5
docs/README.md Normal file
View file

@ -0,0 +1,5 @@
Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/).
Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search.
Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request!

135
docs/administration.md
View file

@ -5,6 +5,7 @@ nav = [
"Installing in production",
"Importing and Exporting Data",
"Versioning",
"Resizing the Cluster",
"Backup/restore",
]
+++
@ -47,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog
#### Importing
The import API expects a csv of RowID,ColumnID's.
The import API expects a csv of rowID,columnID's.
When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
```
pilosa import --sort -i project -f stargazer project-stargazer.csv
@ -64,12 +65,12 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
```
<div class="note">
<p>Note that you must first create a frame with Range Encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
<p>Note that you must first create a frame with range-encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
</div>
#### Exporting
Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID.
Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID.
```
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \
--header "Accept: text/csv"
@ -91,7 +92,106 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ
#### Upgrading
When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes.
To upgrade Pilosa:
1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`.
2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases).
3. Shut down the Pilosa cluster.
4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node.
5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make.
6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes.
##### Version 0.9
Pilosa v0.9 introduces a few compatibility changes that need to be addressed.
**Configuration changes**: These changes need to occur before starting Pilosa v0.9:
1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster.
2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set.
3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds.
**Data directory changes**: These changes need to occur while the cluster is shut down, before starting Pilosa v0.9:
Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`:
1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`.
3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`.
4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position.
5. Copy the file `topology` into the data directories of every node in the cluster, naming it `.topology` (note the dot), e.g. `cp topology ~/.pilosa/.topology` or `scp topology node1:.pilosa/.topology`.
6. Copy the node ID files into the respective node data directories. For example, `node0.id` will be placed on the first node in the `cluster.hosts` list, with the name `.id`. For example: `scp node0.id node0:.pilosa/.id`. Again, it is very important that the ordering you give the nodes with these IDs matches the ordering you had in your existing `cluster.hosts` setting.
**Application changes**:
1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported.
2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed.
3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release.
### Resizing the Cluster
If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process.
#### Adding a Node
You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries.
If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately.
#### Removing a Node
In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself.
To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing an `/id` request to the node:
``` request
curl localhost:10102/id
```
``` response
40a891fa-243b-4d71-ae24-4f5c78a0f4b1
```
If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node:
``` request
curl localhost:10101/status
```
``` response
{
"state":"NORMAL",
"nodes":[
{"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
{"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
{"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
]
}
```
Once you have the ID of the node that you want to remove from the cluster, issue the following request:
```
curl localhost:10101/cluster/resize/remove-node \
-X POST \
-d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}'
```
At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries.
Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator).
#### Aborting a Resize Job
If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node.
For example, if your coordinator node is `localhost:10101`, then you can run:
```
curl localhost:10101/cluster/resize/abort -X POST
```
This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began.
#### Changing the Coordinator
In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator.
```
curl localhost:10101/cluster/resize/set-coordinator \
-X POST \
-d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}'
```
### Backup/restore
@ -112,19 +212,19 @@ Note: This will only work when the replication factor is >= 2
#### Copying data files manually
- To accomplish this goal you will 1st need:
- List of all Indexes on your cluster
- List of all frames in your Indexes
- Max slice per Index, listed in the /status endpoint
- To accomplish this you will first need:
- List of all indexes on your cluster
- List of all frames in your indexes
- Max slice per index, listed in the /status endpoint
- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
- Using the list of slices owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Frame
- copy each owned slice for an existing node to this new node
- Modify the cluster config file to replace the previous node address with the new node address.
- Restart the cluster
- Wait for the 1st sync (10 minutes) to validate Index connections
- Wait for the first sync (10 minutes) to validate Index connections
#### Diagnostics
### Diagnostics
Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
@ -135,7 +235,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
- **NumCPU:** Number of Cores per Node
- **BSIEnabled:** Bit Slice Index Frames in use.
- **TimeQuantumEnabled:** Time Quantum Frames in use.
- **InverseEnabled:** Inverse Frames in use.
- **NumIndexes:** Number of Indexes in the Cluster.
- **NumFrames:** Number of Frames in the Cluster.
- **NumSlices:** Number of Slices in the Cluster.
@ -145,16 +244,16 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
#### Metrics
### Metrics
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
The metrics configuration options are:
- Host to receive events
- Polling interval for runtime metrics
- Metric type (StatsD, Expvar).
- [Host](../configuration/#metric-host): specify host that receives metric events
- [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
- [Service](../configuration/#metric-service): declare type StatsD or Expvar
##### Tags
#### Tags
StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- NodeID
@ -163,7 +262,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- View
- Slice
##### Events
#### Events
We currently track the following events
- **Index:** The creation of a new Index.

252
docs/api-reference.md
View file

@ -13,13 +13,10 @@ nav = []
Returns the schema of all indexes in JSON.
Request:
```
``` request
curl -XGET localhost:10101/index
```
Response:
```
``` response
{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
```
@ -29,13 +26,10 @@ Response:
Returns the schema of the specified index in JSON.
Request:
```
``` request
curl -XGET localhost:10101/index/user
```
Response:
```
``` response
{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
```
@ -45,13 +39,10 @@ Response:
Creates an index with the given name.
Request:
```
``` request
curl -XPOST localhost:10101/index/user
```
Response:
```
``` response
{}
```
@ -61,13 +52,10 @@ Response:
Removes the given index.
Request:
```
``` request
curl -XDELETE localhost:10101/index/user
```
Response:
```
``` response
{}
```
@ -77,15 +65,12 @@ Response:
Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
Request:
```
``` request
curl localhost:10101/index/user/query \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
```
Response:
-d 'Bitmap(frame="language", row=5)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
@ -93,16 +78,14 @@ In order to send protobuf binaries in the request and response, set `Content-Typ
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
Request:
```
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
```
Response:
-d 'Bitmap(frame="language", row=5)'
```
``` response
{
"results":[{"attrs":{},"bits":[100]}],
"columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}]
@ -119,14 +102,14 @@ Creates a frame in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `rangeEnabled` (boolean): Enables range-encoded fields in this frame.
* `fields` (array): List of range-encoded fields.
* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:
* `name` (string): Field name.
* `type` (string): Field type, currently only "int" is supported.
* `min` (int): Minimum value allowed for this field.
@ -134,21 +117,19 @@ Each individual `field` contains the following:
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
Request:
``` request
curl localhost:10101/index/user/frame/language -X POST
```
curl localhost:10101/index/user/frame/language \
-X POST \
-d '{"options": {"inverseEnabled": true}}'
``` response
{}
```
```
``` request
curl localhost:10101/index/repository/frame/stats \
-X POST \
-d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
```
Response:
-d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
```
``` response
{}
```
@ -158,45 +139,10 @@ Response:
Removes the given frame.
Request:
```
``` request
curl -XDELETE localhost:10101/index/user/frame/language
```
Response:
```
{}
```
### Change frame time quantum
`PATCH /index/<index-name>/frame/<frame-name>/time-quantum`
Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame.
The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are:
* (Empty string)
* Y: year
* M: month
* D: day
* H: hour
* YM: year and month
* MD: month and day
* DH: day and hour
* YMD: year, month and day
* MDH: month, day and hour
* YMDH: year, month, day and hour
Request:
```
curl localhost:10101/index/user/frame/language/time-quantum \
-X POST \
-d '{"timeQuantum": "YM"}'
```
Response:
```
``` response
{}
```
@ -211,134 +157,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max`
* `min` (int): Minimum value allowed for this field.
* `max` (int): Maximum value allowed for this field.
Request:
```
``` request
curl localhost:10101/index/repository/frame/stats/field/pullrequests \
-X POST \
-d '{"type": "int", "min": 0, "max": 1000000}'
```
Response:
```
{}
```
### Create input definition
`POST /index/<index-name>/input-definition/<input-definition-name>`
Creates an input definition in the given index with the given name.
The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options:
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa.
* `name` (string): Maps the source data field to actions that process the field's corresponding value.
* `actions` (array): List of actions that will process the field's value.
The `action` describes how the field value will be processed. Each `action` may contain:
* `frame` (string): The Frame that will contain this action's set bits.
* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame.
* `valueDestination` (string): The mapping rule used for this data.
- `value-to-row`: The value should be an integer and will map directly to a RowID.
- `single-row-boolean`: If the value is true set a bit using the `rowid`.
- `mapping`: Map the value to a RowID in the `valueMap`.
* `valueMap` (object): string and integer pairs used to map field values to RowID's.
Request:
```
curl localhost:10101/index/user/input-definition/stargazer-input \
-X POST \
-d '{
"frames":[
{
"name": "language",
"options": {"inverseEnabled": true}
}
],
"fields":[
{
"name": "repo_id",
"primaryKey":true
},
{
"name": "language_id",
"actions":[
{
"frame": "language",
"valueDestination": "mapping",
"valueMap": {
"Go": 5,
"Python": 17,
"C++": 10
}
}
]
}
]
}'
```
Response:
```
{}
```
### Get input definition
`GET /index/<index-name>/input-definition/<input-definition-name>`
Returns the given input definition as JSON.
Request:
```
curl -XGET localhost:10101/index/user/input-definition/stargazer-input
```
Response:
```
{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]}
```
### Remove input definition
`DELETE /index/<index-name>/input-definition/<input-definition-name>`
Removes the given input definition.
Request:
```
curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
```
Response:
```
{}
```
### Process input data
`POST /index/<index-name>/input/<input-definition-name>`
Processes the JSON payload using the given input definition.
The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition.
Request:
```
curl localhost:10101/index/user/input/stargazer-input \
-X POST \
-d '[{"language_id": "Go", "repo_id": 92274475}]'
```
Response:
```
``` response
{}
```
@ -348,13 +172,10 @@ Response:
Returns the hosts in the cluster.
Request:
```
``` request
curl -XGET localhost:10101/hosts
```
Response:
```
``` response
[{"host":":10101"}]
```
@ -364,13 +185,10 @@ Response:
Returns the version of the Pilosa server.
Request:
```
``` request
curl -XGET localhost:10101/version
```
Response:
```
``` response
{"version":"v0.6.0"}
```

3
docs/architecture.md
View file

@ -18,5 +18,8 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B
* After the container storage section is an operation log, of unspecified length.
![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png)
*Pilosa Roaring storage format diagram*
All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.
Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa.

9
docs/client-libraries.md
View file

@ -10,12 +10,13 @@ nav = [
## Client Libraries
This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is not wise to use the output of a hash, or randomly distributed ids with Pilosa.
### Go
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
@ -95,9 +96,9 @@ func main() {
### Python
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions.
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
@ -170,7 +171,7 @@ client.query(stargazer.setbit(99999, 77777))
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.

112
docs/configuration.md
View file

@ -27,24 +27,18 @@ Every command line flag has a corresponding environment variable. The environmen
### Config file
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file:
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file:
```toml
[cluster]
poll-interval = "2m0s"
coordinator = true
replicas = 1
```
Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes:
```toml
[cluster]
hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"]
```
### All Options
#### Anti Entropy Interval
* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync.
* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
* Flag: `--anti-entropy.interval="10m0s"`
* Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"`
* Config:
@ -84,7 +78,18 @@ Any flag that has a value that is a comma separated list on the command line bec
* Config:
```toml
log_path = "/path/to/logfile"
log-path = "/path/to/logfile"
```
#### Verbose
* Description: Enable verbose logging.
* Flag: `--verbose`
* Env: `PILOSA_VERBOSE`
* Config:
```toml
verbose = true
```
#### Max Writes Per Request
@ -110,16 +115,16 @@ Any flag that has a value that is a comma separated list on the command line bec
port = 11101
```
#### Gossip Seed
#### Gossip Seeds
* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`.
* Flag: `--gossip.seed="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEED="localhost:11101"`
* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms.
* Flag: `--gossip.seeds="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101"`
* Config:
```toml
[gossip]
seed = "localhost:11101"
seeds = ["localhost:11101"]
```
#### Gossip Key
@ -133,28 +138,16 @@ Any flag that has a value that is a comma separated list on the command line bec
key = "/var/secret/gossip.key32"
```
#### Cluster Hosts
#### Cluster Coordinator
* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms.
* Flag: `--cluster.hosts="localhost:10101"`
* Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"`
* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
* Flag: `cluster.coordinator`
* Env: `PILOSA_CLUSTER_COORDINATOR`
* Config:
```toml
[cluster]
hosts = ["localhost:10101"]
```
#### Cluster Poll Interval
* Description: Polling interval for cluster.
* Flag: `cluster.poll-interval="1m0s"`
* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"`
* Config:
```toml
[cluster]
poll-interval = "1m0s"
coordinator = true
```
#### Cluster Long Query Time
@ -183,9 +176,8 @@ Any flag that has a value that is a comma separated list on the command line bec
#### Cluster Type
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip].
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip].
* static - Messaging between nodes is disabled. This is primarily used for testing.
* http - Messages are transmitted over HTTP.
* gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip.
* Flag: `cluster.type="gossip"`
* Env: `PILOSA_CLUSTER_TYPE="gossip"`
@ -205,12 +197,12 @@ Any flag that has a value that is a comma separated list on the command line bec
```toml
[profile]
cpu = "/path/to/somewhere"
cpu = "/path/to/somewhere"
```
#### Profile CPU Time
* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set.
* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set.
* Flag: `--profile.cpu-time="30s"`
* Env: `PILOSA_PROFILE_CPU_TIME="30s"
* Config:
@ -219,8 +211,9 @@ Any flag that has a value that is a comma separated list on the command line bec
[profile]
cpu-time = "30s"
```
##### Metric Service
* Description: Which stats service to use (StatsD or ExpVar).
#### Metric Service
* Description: Which stats service to use. Choose from [statsd, expvar, none].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd'
* Config:
@ -230,7 +223,7 @@ Any flag that has a value that is a comma separated list on the command line bec
service = “statsd”
```
##### Metric Host
#### Metric Host
* Description: Address of the StatsD service host.
* Flag: `--metric.host=localhost:8125`
* Env: `PILOSA_METRIC_HOST=localhost:8125'
@ -241,9 +234,9 @@ Any flag that has a value that is a comma separated list on the command line bec
host = "localhost:8125"
```
##### Metric Poll Interval
#### Metric Poll Interval
* Description: Polling interval for runtime metrics.
* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
* Flag: `metric.poll-interval=”0m15s”`
* Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
* Config:
@ -253,9 +246,9 @@ Any flag that has a value that is a comma separated list on the command line bec
poll-interval = "0m15s"
```
##### Metric Diagnostics
#### Metric Diagnostics
* Description: Enable diagnostic reporting. To disable diagnostics set to false.
* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false.
* Flag: `metric.diagnostics`
* Env: `PILOSA_METRIC_DIAGNOSTICS`
* Config:
@ -266,7 +259,7 @@ Any flag that has a value that is a comma separated list on the command line bec
```
##### TLS Certificate
#### TLS Certificate
* Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions.
* Flag: `tls.certificate=/srv/pilosa/certs/server.crt`
@ -278,7 +271,7 @@ Any flag that has a value that is a comma separated list on the command line bec
certificate = "/srv/pilosa/certs/server.crt"
```
##### TLS Certificate Key
#### TLS Certificate Key
* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension.
* Flag: `tls.key=/srv/pilosa/certs/server.key`
@ -290,7 +283,7 @@ Any flag that has a value that is a comma separated list on the command line bec
key = "/srv/pilosa/certs/server.key"
```
##### TLS Skip Verify
#### TLS Skip Verify
* Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial.
* Flag: `tls.skip-verify`
@ -317,8 +310,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = true
#### Node 1
@ -331,8 +323,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
#### Node 2
@ -345,8 +336,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
### Example Cluster Configuration (HTTPS)
@ -365,8 +355,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -384,8 +373,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -403,8 +391,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -426,8 +413,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -445,8 +431,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -464,8 +449,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"

38
docs/data-model.md
View file

@ -24,7 +24,10 @@ Rows and columns can represent anything (they could even represent the same set
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row.
![data model diagram](/img/docs/data-model.svg)
Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen.
![basic data model diagram](/img/docs/data-model.svg)
*Basic data model diagram*
### Index
@ -49,16 +52,18 @@ Row attributes are namespaced at the Frame level.
Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
![ranked frame diagram](/img/docs/frame-ranked.svg)
*Ranked frame diagram*
#### LRU
The LRU cache maintains the most recently accessed Rows.
![lru frame diagram](/img/docs/frame-lru.svg)
*LRU frame diagram*
### Time Quantum
Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
### Attribute
@ -72,42 +77,30 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice.
### View
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
#### Standard
The standard View contains the same Row/Column format as the input data.
#### Inverse
The Inverse View contains the same data with the Row and Column swapped.
For example, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3)
SetBit(frame="A", rowID=11, columnID=3)
SetBit(frame="A", rowID=19, columnID=5)
```
![inverse frame diagram](/img/docs/frame-inverse.svg)
#### Time Quantums
If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
```
![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
*Time quantum frame diagram*
#### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate, aggregate, and range queries on these BSI integers.
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
For example, the following `SetFieldValue()` queries will result in the data described in the illustration below:
@ -120,4 +113,7 @@ SetFieldValue(col=2, frame="A", field1=1)
SetFieldValue(col=3, frame="A", field1=6)
```
![BSI diagram](/img/docs/frame-bsi.svg)
![BSI frame diagram](/img/docs/frame-bsi.svg)
*BSI frame diagram*
Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.

27
docs/examples.md
View file

@ -17,7 +17,7 @@ New York City released an extremely detailed data set of over 1 billion taxi rid
Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
@ -163,7 +163,7 @@ durm := pdk.CustomMapper{
#### Import process
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk/) for more details on this process.
#### Queries
@ -201,11 +201,15 @@ For more examples and details, see this [ipython notebook](https://github.com/pi
### Chemical similarity search
<div class="warning">
This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API.
</div>
#### Overview
The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
```
@ -214,11 +218,11 @@ T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase).
#### Data model
We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
@ -263,16 +267,15 @@ python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
```
First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
```
curl localhost:10101/index/mole \
-X POST \
-d '{"options": {"columnLabel": "position_id"}}'
-X POST
curl localhost:10101/index/mole/frame/fingerprint \
-X POST \
-d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
-d '{"options": {"inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
```
@ -302,7 +305,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a
* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
```python
bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bit_maps = ["Bitmap(col=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bitmap_string = ', '.join(bit_maps)
intersection = "Intersect(%s)" % bitmap_string
mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
@ -312,7 +315,7 @@ Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get a
```python
for m in mole_ids:
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(row=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
existed_mol = False
if len(mol) == len(fp):
found = m
@ -331,7 +334,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN
* Query Pilosa’s TopN to get list of similarity chembl_id
```python
query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
query_string = 'TopN(Bitmap(row=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
```

135
docs/getting-started.md
View file

@ -4,7 +4,6 @@ weight = 3
nav = [
"Starting Pilosa",
"Sample Project",
"Input Definition",
"What's Next?",
]
+++
@ -20,7 +19,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th
### Starting Pilosa
Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa.
Follow the steps in the [Install](../installation/) document to install Pilosa.
Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`):
```
pilosa server
@ -31,49 +30,63 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest
```
Let's make sure Pilosa is running:
```
``` request
curl localhost:10101/status
```
Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}`
``` response
{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]}
```
### Sample Project
In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation.
#### Create the Schema
Note:
The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
```
The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
``` request
curl localhost:10101/schema
```
``` response
{"indexes":null}
```
Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first:
```
``` request
curl localhost:10101/index/repository -X POST
```
``` response
{}
```
Let's create the `stargazer` frame which has user IDs of stargazers as its rows:
```
``` request
curl localhost:10101/index/repository/frame/stargazer \
-X POST \
-d '{"options": {"timeQuantum": "YMD"}}'
```
``` response
{}
```
Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
Next up is the `language` frame, which will contain IDs for programming languages:
``` request
curl localhost:10101/index/repository/frame/language \
-X POST
```
curl localhost:10101/index/repository/frame/language -X POST
``` response
{}
```
#### Import Data From CSV Files
If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo.
Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
@ -95,10 +108,7 @@ docker cp language.csv pilosa:/language.csv
docker exec -it pilosa /pilosa import -i repository -f language /language.csv
```
Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages.
### Input Definition
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
#### Make Some Queries
@ -107,46 +117,117 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d
</div>
Which repositories did user 14 star:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Bitmap(frame="stargazer", rowID=14)'
-d 'Bitmap(frame="stargazer", row=14)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
}
]
}
```
What are the top 5 languages in the sample data:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'TopN(frame="language", n=5)'
```
``` response
{
"results":[
[
{"id":5,"count":119},
{"id":1,"count":50},
{"id":4,"count":48},
{"id":9,"count":31},
{"id":13,"count":25}
]
]
}
```
Which repositories were starred by user 14 and 19:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))'
-d 'Intersect(
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[2,3,362,396,416,461,464,466,470,486]
}
]
}
```
Which repositories were starred by user 14 or 19:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))'
-d 'Union(
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
}
]
}
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))'
-d 'Intersect(
Bitmap(frame="stargazer", row=14),
Bitmap(frame="stargazer", row=19),
Bitmap(frame="language", row=1)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[2,362,416,461]
}
]
}
```
Set user 99999 as a stargazer for repository 77777:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)'
-d 'SetBit(frame="stargazer", column=77777, row=99999)'
```
``` response
{"results":[true]}
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to poor performance, out of memory errors, and more.
### What's Next?

65
docs/glossary.md
View file

@ -6,53 +6,66 @@ nav = []
## Glossary
<strong id="index">Index:</strong> Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
<strong id="column">Column:</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index.
<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
<strong id="row">Row:</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index.
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column).
<strong id="bit">Bit:</strong> A bit is the intersection of a Row and Column.
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
<strong id="bitmap">Bitmap:</strong> The on-disk and in-memory representation of a Row.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
<strong id="roaring-bitmap">Roaring Bitmap:</strong> [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="attribute">Attribute:</strong> Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store.
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index).
<strong id="pql">PQL:</strong> Pilosa Query Language
<strong id="field">[Field](../data-model/#bsi-range-encoding):</strong> A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="index">Index:</strong> The Index represents a data namespace.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
<strong id="frame">Frame:</strong> Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame.
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
<strong id="view">View:</strong> Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a frame and slice in an index.
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="slice">Slice:</strong> Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash.
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="slicewidth">SliceWidth:</strong> This is the default number of columns in a slice.
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
<strong id="maxslice">MaxSlice:</strong> The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries.
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="anti-entropy">Anti-entropy:</strong> A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies.
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a cluster.
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="pql">[PQL](../query-language/):</strong> Pilosa Query Language.
<strong id="topn">TopN:</strong> Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row.
<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="tanimoto">Tanimoto:</strong> Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity.
<strong id="range">[Range](../query-language/#range-queries):</strong>: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
<strong id="protobuf">Protobuf:</strong>: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="range-bsi">[Range (BSI)](../query-language/#range-bsi):</strong>: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
<strong id="toml">TOML:</strong> We use [TOML](https://github.com/toml-lang/toml) for our configuration file format.
<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
<strong id="jump-consistent-hash">Jump Consistent Hash:</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
https://arxiv.org/pdf/1406.2294v1.pdf
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
<strong id="partition">Partition:</strong> The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography.
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="replica">Replica:</strong> A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made).
<strong id="slice">[Slice](../data-model/#slice):</strong> [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
<strong id="slicewidth">SliceWidth:</strong> This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field).
<strong id="tanimoto">[Tanimoto](../examples/#chemical-similarity-search):</strong> Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for time [Range](#range) queries.
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

64
docs/input-definition.md
View file

@ -1,64 +0,0 @@
+++
title = "Input Definition"
weight = 8
nav = [
"Create the Schema",
"Import Data",
]
+++
## Input Definition
This document builds on the data import concepts introduced in [Getting Started](../getting-started/).
Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data.
### Create the Schema
Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations.
Before creating a schema, let's create the repository index first:
```
curl localhost:10101/index/repository -X POST
```
The sample input definition schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started) in the `input_definition.json` file. Download it using:
```
curl -OL https://github.com/pilosa/getting-started/raw/master/input_definition.json
```
Run the following to create the input definition:
```
curl localhost:10101/index/repository/input-definition/stargazer -d @input_definition.json
```
Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition.
We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination:
- value-to-row: The value for this field is used as the `rowID`.
- single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type.
- mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type.
- set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame
### Import Data
The sample data for the input definition we created above is in the `json_input.json` file at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download it using:
```
curl -OL https://github.com/pilosa/getting-started/raw/master/json_input.json
```
Then run the following to import it:
```
curl localhost:10101/index/repository/input/stargazer -d @json_input.json
```
As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID.
The data input above is equivalent to the following `SetBit()` operations:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=91720568, rowID=513114)
SetBit(frame="stargazer", columnID=91720568, rowID=513114, timestamp="2017-05-18T20:40")
SetBit(frame="language", columnID=91720568, rowID=5)
SetBit(frame="language", columnID=95122322, rowID=17)
'
```

39
docs/installation.md
View file

@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
1. Download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-darwin-amd64.tar.gz
```
Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz
tar xfz pilosa-v0.9.0-darwin-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin
cp -i pilosa-v0.9.0-darwin-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
@ -134,19 +134,25 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa):
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -203,7 +209,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
docker version
```
If you don't see the server listed, start the Docker application.
If you don't see the server listed, start the Docker application.
3. Pull the official Pilosa image from Docker Hub:
```
@ -228,19 +234,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
1. To install the latest version of Pilosa, download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-linux-amd64.tar.gz
```
Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.8.3-linux-amd64.tar.gz
tar xfz pilosa-v0.9.0-linux-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin
cp -i pilosa-v0.9.0-linux-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
@ -287,19 +293,26 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo:
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```

2
docs/introduction.md
View file

@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w
"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.
Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.

110
docs/pdk.md
View file

@ -2,66 +2,72 @@
title = "PDK"
weight = 11
nav = [
"Examples and Executables",
"Library",
"Examples",
]
+++
## PDK
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly.
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively.
### Examples and Executables
Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here.
#### Kafka
`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
record from Kafka is assigned a Pilosa column, and each value in a record is
assigned a row or field. Frame and field names are built from the "path" through
the record to arrive at that field. For example:
```json
{
"name": "jill",
"favorite_foods": ["corn chips", "chipotle dip"],
"location": {
"city": "Austin",
"state": "Texas",
"latitude": 3754,
"longitude": 4526
},
"active": true,
"age": 27
}
```
This JSON object would result in the following Pilosa schema:
| Name | Field | Type | Size/Min | Max |
|----------------|-----------|--------|----------|------------|
| name | | ranked | 100000 | |
| favorite_foods | | ranked | 100000 | |
| default | | Ranked | 100000 | |
| | age | int | 0 | 2147483647 |
| location | | ranked | 1000 | |
| | latitude | int | 0 | 2147483647 |
| | longitude | int | 0 | 2147483647 |
| location-city | | ranked | 100000 | |
| location-state | | ranked | 100000 | |
All frames are created as ranked frames by default, and fields are created with
a minmum size of zero and a fixed maximum of 2147483647. Fields at the top level
are created in the default frame. Frames are a dash-separated concatenation of
all key values in the path - you can see this with frames like location-city.
Most the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
kafka topics, kafka group, etc.), but there are a few options that give some
control over the way data is indexed, and ingestion performance.
* `--batch-size`: The batch size control how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state".
* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.
The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary.
### Library
#### Mapping
For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation.
Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts:
* an enumeration of field names
* a list of parsers that are used to parse strings in the CSV to values
* a list of commonly used, named, mapper functions
* a list of ParserMappers - objects that encapsulate all of the work related to a single frame.
This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK.
With this definition available, the PDK tool can run the import, which consists of these steps:
- create the index
- create all frames
- for each CSV file, read all rows
- for each CSV record:
- generate a columnID
- apply all ParserMappers, generating a list of (frame, ID) pairs
- set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID)
The process is summarized in this flowchart:
![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg)
Some of the simple mapper functions available with PDK include:
* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year.
* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11].
* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6].
* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23].
* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins.
* BoolMapper: Maps a boolean value to the range [0, 1].
* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain.
* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small.
* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets.
* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket.
* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example.
* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper.
### Examples
Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help <example>` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality.
<!--
#### Net
A detailed discussion of using Pilosa to index network traffic data is available [here - TODO]link blog post). This will discuss the implementation of `pdk net` as it relates to the use of the PDK library tools.
-->

219
docs/query-language.md
View file

@ -13,7 +13,7 @@ nav = [
### Overview
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
```
{"results":[...]}
@ -21,35 +21,34 @@ This section will provide a detailed reference and examples for the Pilosa Query
There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type.
The default row label is `rowID`, and the default column label is `columnID`. Changing these defaults is deprecated and this feature will be removed in a future release.
#### Conventions
* Angle Brackets `<>` denote required arguments
* Square Brackets `[]` denote optional arguments
* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`)
* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`)
##### Examples
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data.
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
```
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=10, rowID=1)'
-d 'SetBit(frame="stargazer", col=10, row=1)'
```
``` response
{"results":[true]}
```
#### Arguments and Types
* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated.
* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated.
* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
* `UINT` An unsigned integer (e.g. 42839)
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
* `ATTR_VALUE` Can be a string, float, integer, or bool.
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range`
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
### Write Operations
@ -59,7 +58,7 @@ curl localhost:10101/index/repository/query \
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
SetBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
```
@ -77,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
**Examples:**
```
SetBit(frame="stargazer", repo_id=10, rowID=1)
SetBit(frame="stargazer", col=10, row=1)
```
This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10.
SetBit also supports providing a timestamp. To write the date that a user starred a repository.
```
SetBit(frame="stargazer", repo_id=10, rowID=1, timestamp="2016-01-01T00:00")
SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
```
Setting multiple bits in a single request:
```
SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2)
SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2)
```
#### SetRowAttrs
**Spec:**
```
SetRowAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
SetRowAttrs(<frame=STRING>, <row=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
@ -112,13 +111,13 @@ SetRowAttrs queries always return `null` upon success.
**Examples:**
```
SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true)
SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true)
```
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
```
SetRowAttrs(frame="stargazer", rowID=10, username=null)
SetRowAttrs(frame="stargazer", row=10, username=null)
```
Delete username value for user 10.
@ -128,7 +127,7 @@ Delete username value for user 10.
**Spec:**
```
SetColumnAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
SetColumnAttrs(<frame=STRING>, <row=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
@ -144,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu
**Examples:**
```
SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true)
SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true)
```
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`.
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`.
```
SetColumnAttrs(columnID=10, url=null)
SetColumnAttrs(col=10, url=null)
```
Delete url value for repo 10.
@ -161,8 +160,8 @@ Delete url value for repo 10.
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
[timestamp=TIMESTAMP])
ClearBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
```
**Description:**
@ -178,10 +177,34 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
**Examples:**
```
ClearBit(frame="stargazer", columnID=10, rowID=1)
ClearBit(frame="stargazer", col=10, row=1)
```
Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame.
Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame.
#### SetFieldValue
**Spec:**
```
SetFieldValue(<col=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`.
**Result Type:** null
SetFieldValue returns `null` upon success.
**Examples:**
Set the number of pull requests of repository 10.
```
SetFieldValue(col=10, frame="stats", pullrequests=2)
```
### Read Operations
@ -191,12 +214,12 @@ Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer fr
**Spec:**
```
Bitmap(<frame=STRING>, (<ROW_LABEL=UINT> | <COL_LABEL>=UINT))
Bitmap(<frame=STRING>, (<rowL=UINT> | <col>=UINT))
```
**Description:**
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column.
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column.
**Result Type:** object with attrs and bits.
@ -206,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
Query all repositories that user 1 has starred.
```
Bitmap(frame="stargazer", rowID=1)
Bitmap(frame="stargazer", row=1)
```
Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
@ -263,7 +286,7 @@ attrs will always be empty
Query repositories which have been starred by two users.
```
Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
```
Returns `{"attrs":{},"bits":[10]}`.
@ -290,7 +313,7 @@ attrs will always be empty
Query repositories which have been starred by one user and not another.
```
Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2))
Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2))
```
Return `{"results":[{"attrs":{},"bits":[20]}]}`
@ -298,13 +321,41 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}`
* bits are repositories that were starred by user 1 BUT NOT user 2
```
Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1))
Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1))
```
Return `{"attrs":{},"bits":[30]}`
* Bits are repositories that were starred by user 2 BUT NOT user 1
#### Xor
**Spec:**
```
Xor(<BITMAP_CALL>, [BITMAP_CALL ...])
```
**Description:**
Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it.
**Result Type:** object with attrs and bits
attrs will always be empty
**Examples:**
Query repositories which have been starred by two users.
```
Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
```
Returns `{"attrs":{},"bits":[30]}`.
* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
#### Count
**Spec:**
@ -322,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in.
Query the number of repositories to which a user has contributed.
```
Count(Bitmap(frame="stargazer", rowID=1))
Count(Bitmap(frame="stargazer", row=1))
```
Return `2`
@ -335,13 +386,12 @@ Return `2`
```
TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
[inverse=true], [<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
[<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
```
**Description:**
Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse).
The `field` and `filters` arguments work together to only return Bitmaps which
have the attribute specified by `field` with one of the values specified in
`filters`.
@ -349,6 +399,7 @@ have the attribute specified by `field` with one of the values specified in
**Result Type:** array of key/count objects
**Caveats:**
* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order.
* Frames with cache type lru will maintain an LRU (Least Recently Used) cache, thus a TopN() query on this type of frame will return bitmaps sorted in order of most recently set bit.
* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN() queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
@ -367,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}
* count is amount of repositories
* Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository.
```
TopN(frame="stargazer", inverse=true)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]`
* key is a repository ID
* count is amount of users
* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame.
```
TopN(frame="stargazer", n=2)
```
@ -386,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]`
* Results are the top two users sorted by number of repositories they've starred in descending order.
```
TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2)
TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
@ -398,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
**Spec:**
```
Range(<frame=STRING>, <ROW_LABEL=UINT>,
Range(<frame=STRING>, <row=UINT>,
<start=TIMESTAMP>, <end=TIMESTAMP>)
```
@ -414,7 +455,7 @@ between the given `start` and `end` timestamps.
When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range.
```
Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
```
Returns `{{"attrs":{},"bits":[10]}`
@ -441,7 +482,7 @@ Returns bits that are true for the comparison operator.
**Examples:**
In our source data, commitactivity was counted over the last year.
The following greater-than Range query returns all repositories having more than 100 commits.
The following greater-than `Range` query returns all repositories having more than 100 commits.
```
Range(frame="stats", commitactivity > 100)
@ -471,6 +512,56 @@ Range(frame="stats", commitactivity >< [100, 200])
This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
#### Min
**Spec:**
```
Min([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered.
**Result Type:** object with the min and count of columns containing the min value.
**Examples:**
Query the size of all repositories.
```
Min(frame="stats", field="diskusage")
```
Return `{"min":4,"count":2}`
* Result is the smallest repository in kilobytes, plus the number of repositories of that size.
#### Max
**Spec:**
```
Max([BITMAP_CALL], <frame=STRING>, <field=STRING>)
```
**Description:**
Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
**Result Type:** object with the max and count of columns containing the max value.
**Examples:**
Query the size of all repositories.
```
Max(frame="stats", field="diskusage")
```
Return `{"max":88,"count":13}`
* Result is the largest repository in kilobytes, plus the number of repositories of that size.
#### Sum
**Spec:**
@ -481,7 +572,7 @@ Sum([BITMAP_CALL], <frame=STRING>, <field=STRING>)
**Description:**
Returns the count and computed sum of all bitmap encoded integer values across the `field` in this `frame`. The optional Bitmap call filters the bits used in this computation.
Returns the count and computed sum of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
**Result Type:** object with the computed sum and count of the bitmap field.
@ -495,27 +586,3 @@ Sum(frame="stats", field="diskusage")
Return `{"sum":10,"count":3}`
* Result is the size of all repositories in kilobytes, plus the number of repositories.
#### SetFieldValue
**Spec:**
```
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
**Result Type:** null
SetFieldValue returns `null` upon success.
**Examples:**
Set the number of pull requests of repository 10.
```
SetFieldValue(col=10, frame="stats", pullrequests=2)
```

258
docs/tutorials.md
View file

@ -10,6 +10,16 @@ nav = [
## Tutorials
<div class="note">
<!-- this is html because there is a problem putting a list inside a shortcode -->
Some of our tutorials work better as standalone repos, since you can <code>git clone</code> the instructions, code, and data all at once. Officially supported tutorials are listed here.<br />
<br />
<ul>
<li><a href="https://github.com/pilosa/cosmosa">Run Pilosa with Microsoft's Azure Cosmos DB</a></li>
</ul>
</div>
### Setting Up a Secure Cluster
#### Introduction
@ -20,7 +30,7 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma
#### Installing Pilosa and Creating the Directory Structure
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source).
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
```
@ -45,6 +55,7 @@ openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.cr
```
The command above creates two files in the current directory:
* `pilosa.local.crt` is the SSL certificate.
* `pilosa.local.key` is the private key file which must be kept as secret.
@ -59,6 +70,8 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand
Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes.
One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated.
Create `node1.config.toml` in the project directory and paste the following in it:
```toml
@ -68,7 +81,7 @@ data-dir = "node1_data"
bind = "https://01.pilosa.local:10501"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
coordinator = true
[tls]
certificate = "pilosa.local.crt"
@ -76,7 +89,7 @@ key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 15000
key = "pilosa.local.gossip32"
```
@ -89,16 +102,13 @@ Create `node2.config.toml` in the project directory and paste the following in i
data-dir = "node2_data"
bind = "https://02.pilosa.local:10502"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
[tls]
certificate = "pilosa.local.crt"
key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 16000
key = "pilosa.local.gossip32"
```
@ -111,26 +121,24 @@ Create `node3.config.toml` in the project directory and paste the following in i
data-dir = "node3_data"
bind = "https://03.pilosa.local:10503"
[cluster]
hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"]
[tls]
certificate = "pilosa.local.crt"
key = "pilosa.local.key"
skip-verify = true
[gossip]
seed = "01.pilosa.local:15000"
seeds = ["01.pilosa.local:15000"]
port = 17000
key = "pilosa.local.gossip32"
```
Here is some explanation of the configuration items:
* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings.
* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings.
* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
#### Final Touches Before Running the Cluster
@ -177,38 +185,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status
The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
All nodes should be in the `UP` state:
```
{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}}
All nodes should be in the `NORMAL` state:
``` response
{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]}
```
#### Running Queries
Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d ''
```
``` response
{}
```
This will create index `sample-index` with default options. Let's create the frame now:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d ''
```
``` response
{}
```
We just created frame `sample-frame` with default options.
Let's run a `SetBit` query:
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)'
```
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)'
``` response
{"results":[true]}
```
Confirm that the bit was indeed set:
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
```
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
The same response should be returned when querying other nodes in the cluster:
``` request
curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)'
```
curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
#### What's Next?
@ -220,27 +243,22 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
```
``` request
curl localhost:10101/index/patients \
-X POST
```
Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient.
```
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{"rangeEnabled": true}}'
``` response
{}
```
In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
```
``` request
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{
"rangeEnabled": true,
"fields": [
{"name": "age", "type": "int", "min": 0, "max": 120},
{"name": "weight", "type": "int", "min": 0, "max": 500},
@ -248,16 +266,22 @@ curl localhost:10101/index/patients/frame/measurements \
]
}}'
```
``` response
{}
```
If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field).
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
This query sets the age, weight, and t-cell count for the patient with ID `1` in our system:
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)'
-d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)'
```
``` response
{"results":[null]}
```
In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
@ -282,39 +306,80 @@ pilosa import -i patients -f measurements --field age ages.csv
Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field.
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Range(frame="measurements", age > 40)'
```
You should get the following results:
```
``` response
{"results":[{"attrs":{},"bits":[2,6,9]}]}
```
You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(frame="measurements", field="age")'
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
```
``` response
{"results":[{"sum":377,"count":9}]}
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40.
```
You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
```
``` response
{"results":[{"sum":191,"count":3}]}
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
To find the minimum age of all patients, run a `Min` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(frame="measurements", field="age")'
```
``` response
{"results":[{"min":19,"count":1}]}
```
The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
``` response
{"results":[{"min":57,"count":1}]}
```
To find the maximum age of all patients, run a `Max` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(frame="measurements", field="age")'
```
``` response
{"results":[{"max":71,"count":1}]}
```
The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")'
```
``` response
{"results":[{"max":34,"count":1}]}
```
### Storing Row and Column Attributes
@ -323,95 +388,102 @@ Notice in this case that the count is only `3` because of the `age > 40` filter
Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data.
First, create an index called `books` to use for this tutorial:
```
``` request
curl localhost:10101/index/books \
-X POST
```
``` response
{}
```
Next, create a frame in the `books` index called `members` which will represent library members who have read books.
```
``` request
curl localhost:10101/index/books/frame/members \
-X POST \
-d '{}'
```
``` response
{}
```
Now, let's add some books to our index.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972)
SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000)
SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(columnID=5, name="The Forever War", year=2008)'
-d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(col=2, name="No Name in the Street", year=1972)
SetColumnAttrs(col=3, name="The Tipping Point", year=2000)
SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(col=5, name="The Forever War", year=2008)'
```
``` response
{"results":[null,null,null,null,null]}
```
And add some members.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith")
SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")'
-d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith")
SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")'
```
``` response
{"results":[null,null,null,null,null]}
```
At this point we can query one of the `member` records by querying that row.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
You should get the following result set:
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
```
Now let's add some data to the matrix such that each pair represents a member who has read that book.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetBit(frame="members", rowID=10001, columnID=3)
SetBit(frame="members", rowID=10001, columnID=5)
SetBit(frame="members", rowID=10002, columnID=1)
SetBit(frame="members", rowID=10002, columnID=2)
SetBit(frame="members", rowID=10002, columnID=4)
SetBit(frame="members", rowID=10003, columnID=3)
SetBit(frame="members", rowID=10004, columnID=4)
SetBit(frame="members", rowID=10004, columnID=5)
SetBit(frame="members", rowID=10005, columnID=1)
SetBit(frame="members", rowID=10005, columnID=2)
SetBit(frame="members", rowID=10005, columnID=3)
SetBit(frame="members", rowID=10005, columnID=4)
SetBit(frame="members", rowID=10005, columnID=5)'
-d 'SetBit(frame="members", row=10001, col=3)
SetBit(frame="members", row=10001, col=5)
SetBit(frame="members", row=10002, col=1)
SetBit(frame="members", row=10002, col=2)
SetBit(frame="members", row=10002, col=4)
SetBit(frame="members", row=10003, col=3)
SetBit(frame="members", row=10004, col=4)
SetBit(frame="members", row=10004, col=5)
SetBit(frame="members", row=10005, col=1)
SetBit(frame="members", row=10005, col=2)
SetBit(frame="members", row=10005, col=3)
SetBit(frame="members", row=10005, col=4)
SetBit(frame="members", row=10005, col=5)'
```
``` response
{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
```
Now pull the record for `Sue Perkins` again.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
```
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute.
-d 'Bitmap(frame="members", row=10002)'
```
``` response
{
"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}],
"columnAttrs":[
@ -421,15 +493,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt
]
}
```
The `book` attributes are included in the result set at the `columnAttrs` attribute.
Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members:
```
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))'
```
-d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))'
```
``` response
{
"results":[{"attrs":{},"bits":[4]}],
"columnAttrs":[

13
docs/webui.md
View file

@ -9,18 +9,20 @@ nav = [
## WebUI
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101)
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
This can be used for constructing queries and viewing the cluster status.
### Console
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
The Console will keep a record of each query and its result with the latest query on top.
![console](/img/docs/webui-console.png)
![webUI console screenshot](/img/docs/webui-console.png)
*WebUI console screenshot*
In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
@ -30,10 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix
- `:create frame <framename>`
- `:delete frame <framename>`
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference).
Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
- `:create index <indexname> timeQuantum=YM`
- `:create frame <framename> inverseEnabled=true cacheSize=10000`
- `:create frame <framename> cacheSize=10000`
### Cluster Admin

55
event.go Normal file
View file

@ -0,0 +1,55 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// NodeEventType are the types of events that can be sent from the
// ChannelEventDelegate.
type NodeEventType int
const (
NodeJoin NodeEventType = iota
NodeLeave
NodeUpdate
)
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
}
// EventHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type EventHandler interface {
ReceiveEvent(e *NodeEvent) error
}
// EventReceiver is the interface for the object which will listen for and
// decode broadcast messages before passing them to pilosa to handle. The
// implementation of this could be an http server which listens for messages,
// gets the protobuf payload, and then passes it to
// EventHandler.ReceiveMessage.
type EventReceiver interface {
// Start starts listening for broadcast messages - it should return
// immediately, spawning a goroutine if necessary.
Start(EventHandler) error
}
type nopEventReceiver struct{}
func (n *nopEventReceiver) Start(e EventHandler) error { return nil }
// NopEventReceiver is a no-op implementation of the EventReceiver.
var NopEventReceiver = &nopEventReceiver{}

400
executor.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
@ -24,6 +23,7 @@ import (
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
// DefaultFrame is the frame used if one is not specified.
@ -33,6 +33,9 @@ const (
// MinThreshold is the lowest count to use in a Top-N operation when
// looking for additional id/count pairs.
MinThreshold = 1
columnLabel = "col"
rowLabel = "row"
)
// Executor recursively executes calls in a PQL query across all slices.
@ -40,8 +43,7 @@ type Executor struct {
Holder *Holder
// Local hostname & cluster configuration.
Scheme string
Host string
Node *Node
Cluster *Cluster
// Client used for remote requests.
@ -81,35 +83,32 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// MaxSlice can differ between inverse and standard views, so we need
// to send queries to different slices based on orientation.
var inverseSlices []uint64
rowLabel := DefaultRowLabel
columnLabel := DefaultColumnLabel
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Determine slices and inverseSlices for use in e.executeCall().
if needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// If slices are specified, then use that value for slices or
// inverseSlices. If slices aren't specified, then include all of them.
if len(slices) > 0 {
// For inverse queries, the values of `slices` provided to the Execute() method
// on the remote node actually represents inverseSlices.
inverseSlices = slices
} else if needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
// Fetch column label from index.
columnLabel = idx.ColumnLabel()
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
}
@ -121,7 +120,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if call.SupportsInverse() && needsSlices {
// Fetch frame & row label based on argument.
frame, _ := call.Args["frame"].(string)
@ -132,7 +130,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel = f.RowLabel()
// If this call is to an inverse frame send to a different list of slices.
if call.IsInverse(rowLabel, columnLabel) {
@ -151,9 +148,8 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
if err := e.validateCallArgs(c); err != nil {
return nil, err
return nil, errors.Wrap(err, "validating args")
}
indexTag := fmt.Sprintf("index:%s", index)
// Special handling for mutation and top-n calls.
@ -161,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
case "Min":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMin(ctx, index, c, slices, opt)
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMax(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -203,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
}
// executeSum executes a Sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) {
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return SumCount{}, errors.New("Sum(): frame required")
return ValCount{}, errors.New("Sum(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return SumCount{}, errors.New("Sum(): field required")
return ValCount{}, errors.New("Sum(): field required")
}
if len(c.Children) > 1 {
return SumCount{}, errors.New("Sum() only accepts a single bitmap input")
return ValCount{}, errors.New("Sum() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
@ -221,18 +223,88 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
other, _ := prev.(ValCount)
return other.Add(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return SumCount{}, err
return ValCount{}, err
}
other, _ := result.(SumCount)
other, _ := result.(ValCount)
if other.Count == 0 {
return SumCount{}, nil
return ValCount{}, nil
}
return other, nil
}
// executeFieldMin executes a Min() call.
func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Min(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Min() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMinSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Smaller(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
// executeFieldMax executes a Max() call.
func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Max(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
if len(c.Children) > 1 {
return ValCount{}, errors.New("Max() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMaxSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Larger(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
other, _ := result.(ValCount)
if other.Count == 0 {
return ValCount{}, nil
}
return other, nil
}
@ -269,11 +341,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else {
idx := e.Holder.Index(index)
if idx != nil {
columnLabel := idx.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting column attrs")
}
bm.Attrs = attrs
} else if err != nil {
@ -281,14 +352,13 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else {
frame, _ := c.Args["frame"].(string)
if fr := idx.Frame(frame); fr != nil {
rowLabel := fr.RowLabel()
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting row")
}
attrs, err := fr.RowAttrStore().Attrs(rowID)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting row attrs")
}
bm.Attrs = attrs
}
@ -324,13 +394,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
}
}
// executeSumCountSlice executes calculates the sum & count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) {
// executeSumCountSlice calculates the sum and count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return SumCount{}, err
return ValCount{}, errors.Wrap(err, "executing bitmap call")
}
filter = bm
}
@ -340,29 +410,107 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return SumCount{}, nil
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return SumCount{}, nil
return ValCount{}, nil
}
view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if view == nil {
return SumCount{}, nil
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth())
if err != nil {
return SumCount{}, err
return ValCount{}, errors.Wrap(err, "computing sum")
}
return SumCount{
Sum: int64(vsum) + (int64(vcount) * field.Min),
return ValCount{
Val: int64(vsum) + (int64(vcount) * field.Min),
Count: int64(vcount),
}, nil
}
// executeFieldMinSlice calculates the min for fields on a slice.
func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmin) + field.Min,
Count: int64(fcount),
}, nil
}
// executeFieldMaxSlice calculates the max for fields on a slice.
func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return ValCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmax) + field.Min,
Count: int64(fcount),
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
@ -379,7 +527,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
// Execute original query.
pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "finding top results")
}
// If this call is against specific ids, or we didn't get results,
@ -396,7 +544,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "retrieving full counts")
}
if n != 0 && int(n) < len(trimmedList) {
@ -526,7 +674,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
if idx == nil {
return nil, ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Fetch frame & row label based on argument.
frame, _ := c.Args["frame"].(string)
@ -537,7 +684,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
// Return an error if both the row and column label are specified.
rowID, rowOK, rowErr := c.UintArg(rowLabel)
@ -607,14 +753,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
if idx == nil {
return nil, ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Retrieve base frame.
f := idx.Frame(frame)
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
// Read row & column id.
columnID, columnOK, err := c.UintArg(columnLabel)
@ -739,7 +883,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
predicates, err := cond.IntSliceValue()
if err != nil {
return nil, err
return nil, errors.Wrap(err, "getting condition value")
}
// Only support two integers for the between operation.
@ -801,7 +945,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return NewBitmap(), nil
}
// LT[E] and GT[E] should return all not-null if selected range fully encompases valid field range.
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range.
if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) ||
(cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) {
return frag.FieldNotNull(field.BitDepth())
@ -905,10 +1049,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, ErrFrameNotFound
}
// Retrieve labels.
columnLabel := idx.ColumnLabel()
rowLabel := f.RowLabel()
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -955,9 +1095,9 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
if err != nil {
return false, err
@ -999,10 +1139,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, ErrFrameNotFound
}
// Retrieve labels.
columnLabel := idx.ColumnLabel()
rowLabel := f.RowLabel()
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -1060,9 +1196,9 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
if err != nil {
return false, err
@ -1094,13 +1230,6 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
return errors.New("SetFieldValue() frame required")
}
// Retrieve column label.
idx := e.Holder.Index(index)
if idx == nil {
return ErrIndexNotFound
}
columnLabel := idx.ColumnLabel()
// Retrieve frame.
frame := e.Holder.Frame(index, frameName)
if frame == nil {
@ -1141,7 +1270,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1172,7 +1301,6 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
if frame == nil {
return ErrFrameNotFound
}
rowLabel := frame.RowLabel()
// Parse labels.
rowID, ok, err := c.UintArg(rowLabel)
@ -1199,7 +1327,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1233,7 +1361,6 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
@ -1286,7 +1413,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1314,28 +1441,18 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
return ErrIndexNotFound
}
var colName string
id, okID, errID := c.UintArg("id")
if errID != nil || !okID {
// Retrieve columnLabel
columnLabel := idx.columnLabel
col, okCol, errCol := c.UintArg(columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
}
id = col
colName = columnLabel
} else {
colName = "id"
col, okCol, errCol := c.UintArg(columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, colName)
delete(attrs, columnLabel)
delete(attrs, "frame")
// Set attributes.
if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil {
if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil {
return err
}
idx.Stats.Count("SetProfileAttrs", 1, 1.0)
@ -1345,7 +1462,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1372,13 +1489,8 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
Slices: slices,
Remote: true,
}
uri, err := NewURIFromAddress(node.Host)
if err != nil {
return nil, err
}
uri.SetScheme(node.Scheme)
ctx = context.WithValue(ctx, "uri", uri)
pb, err := e.client.ExecuteQuery(ctx, index, pbreq)
pb, err := e.client.QueryNode(ctx, &node.URI, index, pbreq)
if err != nil {
return nil, err
}
@ -1396,7 +1508,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
switch call.Name {
case "Average", "Sum":
v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
@ -1426,7 +1538,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m
loop:
for _, slice := range slices {
for _, node := range e.Cluster.FragmentNodes(index, slice) {
for _, node := range e.Cluster.SliceNodes(index, slice) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
continue loop
@ -1456,12 +1568,12 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.Nodes).Clone()
} else {
nodes = []*Node{e.Cluster.NodeByHost(e.Host)}
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil {
return nil, err
return nil, errors.Wrap(err, "starting mapper")
}
// Iterate over all map responses and reduce.
@ -1512,7 +1624,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
resp := mapResponse{node: n, slices: nodeSlices}
// Send local slices to mapper, otherwise remote exec.
if n.Host == e.Host {
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
@ -1634,29 +1746,51 @@ func needsSlices(calls []*pql.Call) bool {
return false
}
// SumCount represents a grouping of sum & count for Sum() and Average() calls.
type SumCount struct {
Sum int64 `json:"sum"`
// ValCount represents a grouping of sum & count for Sum() and Average() calls.
type ValCount struct {
Val int64 `json:"value"`
Count int64 `json:"count"`
}
func (sc *SumCount) Add(other SumCount) SumCount {
return SumCount{
Sum: sc.Sum + other.Sum,
Count: sc.Count + other.Count,
func (vc *ValCount) Add(other ValCount) ValCount {
return ValCount{
Val: vc.Val + other.Val,
Count: vc.Count + other.Count,
}
}
func encodeSumCount(sc SumCount) *internal.SumCount {
return &internal.SumCount{
Sum: sc.Sum,
Count: sc.Count,
func encodeValCount(vc ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
func decodeSumCount(pb *internal.SumCount) SumCount {
return SumCount{
Sum: pb.Sum,
func decodeValCount(pb *internal.ValCount) ValCount {
return ValCount{
Val: pb.Val,
Count: pb.Count,
}
}
// Smaller returns the smaller of the two ValCounts.
func (vc *ValCount) Smaller(other ValCount) ValCount {
if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
// Larger returns the larger of the two ValCounts.
func (vc *ValCount) Larger(other ValCount) ValCount {
if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) {
return other
}
return ValCount{
Val: vc.Val,
Count: vc.Count,
}
}

311
executor_test.go
View file

@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -101,7 +101,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -122,7 +122,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -154,7 +154,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -184,7 +184,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -217,7 +217,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -233,7 +233,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -262,7 +262,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
// Create frames.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2},
@ -293,9 +292,9 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
// Set field values.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=100, frame=f, field0=10)`), nil, nil); err != nil {
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100},
},
@ -340,28 +338,28 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
t.Run("ErrFrameRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldValue", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'columnID' required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFieldValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(columnID=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` {
t.Fatalf("unexpected error: %s", err)
}
})
@ -384,16 +382,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -419,15 +417,15 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=1)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(frame=f, rowID=10, columnID=0)
SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=other, rowID=0, columnID=0)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=1)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(frame=f, row=10, col=0)
SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=other, row=0, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -543,7 +541,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -590,7 +588,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -599,6 +597,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}
}
// Ensure Min() and Max() queries can be executed.
func TestExecutor_Execute_MinMax(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=3)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=1, col=1)
SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=0)
SetFieldValue(frame=f, foo=-5, col=1)
SetFieldValue(frame=f, foo=-5, col=2)
SetFieldValue(frame=f, foo=10, col=3)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Min(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter string
exp int64
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Max(frame=f, field=foo)`
} else {
pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) {
t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result))
}
}
})
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
@ -611,7 +701,6 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@ -621,7 +710,6 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
@ -630,15 +718,15 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
SetFieldValue(frame=f, foo=20, bar=2000, col=0)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -646,15 +734,15 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) {
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -679,22 +767,22 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00")
SetBit(frame=f, rowID=1, columnID=5, timestamp="2000-02-01T00:00")
SetBit(frame=f, rowID=1, columnID=6, timestamp="2001-01-01T00:00")
SetBit(frame=f, rowID=1, columnID=7, timestamp="2002-01-01T02:00")
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00")
SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-30T00:00")
SetBit(frame=f, rowID=1, columnID=2, timestamp="2002-02-01T00:00")
SetBit(frame=f, rowID=10, columnID=2, timestamp="2001-01-01T00:00")
SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00")
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -703,7 +791,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Run("Inverse", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -723,7 +811,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
@ -733,7 +820,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
@ -742,7 +828,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100},
},
@ -751,17 +836,17 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=50)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
SetFieldValue(frame=edge, foo=100, columnID=0)
SetFieldValue(frame=edge, foo=-100, columnID=1)
SetFieldValue(frame=f, foo=20, bar=2000, col=50)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
SetFieldValue(frame=edge, foo=100, col=0)
SetFieldValue(frame=edge, foo=-100, col=1)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -897,14 +982,18 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Scheme = "http"
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(frame="f", rowID=10)` {
} else if query.String() != `Bitmap(frame="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
@ -923,11 +1012,11 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -941,7 +1030,13 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -951,12 +1046,12 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create local executor data. The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -971,14 +1066,20 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` {
} else if query.String() != `SetBit(col=2, frame="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -988,7 +1089,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Create local executor data.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
@ -996,7 +1097,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1017,14 +1118,20 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` {
} else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1034,7 +1141,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create local executor data.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
// Create frame.
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
@ -1044,7 +1151,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1064,7 +1171,13 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int
@ -1102,7 +1215,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create local executor data on slice 2 & 4.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
@ -1141,11 +1254,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// SetColumnAttrs call should exclude the frame attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=10)"), nil, nil)
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', columnID=10, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1158,11 +1271,11 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
}
// SetColumnAttrs call should not break if frame is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', rowID=1, columnID=20)"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(columnID=20, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}

42
filesystem.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"net/http"
)
// Ensure nopFileSystem implements interface.
var _ FileSystem = &nopFileSystem{}
// FileSystem represents an interface for a WebUI file system.
type FileSystem interface {
New() (http.FileSystem, error)
}
func init() {
NopFileSystem = &nopFileSystem{}
}
// NopFileSystem represents a FileSystem that returns an error if called.
var NopFileSystem FileSystem
type nopFileSystem struct{}
// New is a no-op implementation of FileSystem New method.
func (n *nopFileSystem) New() (http.FileSystem, error) {
return nil, fmt.Errorf("file system not implemented")
}

135
fragment.go
View file

@ -20,14 +20,12 @@ import (
"bytes"
"container/heap"
"context"
"crypto/sha1"
"encoding/binary"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sort"
@ -36,6 +34,8 @@ import (
"time"
"unsafe"
"github.com/cespare/xxhash"
"math"
"github.com/gogo/protobuf/proto"
@ -102,12 +102,12 @@ type Fragment struct {
// so that they can be mmapped and heap utilization can be kept low.
MaxOpN int
// Writer used for out-of-band log entries.
LogOutput io.Writer
// Logger used for out-of-band log entries.
Logger Logger
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
RowAttrStore *AttrStore
RowAttrStore AttrStore
stats StatsClient
}
@ -123,8 +123,8 @@ func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
LogOutput: ioutil.Discard,
MaxOpN: DefaultFragmentMaxOpN,
Logger: NopLogger,
MaxOpN: DefaultFragmentMaxOpN,
stats: NopStatsClient,
}
@ -255,6 +255,7 @@ func (f *Fragment) openCache() error {
f.cache = NewLRUCache(f.CacheSize)
case CacheTypeNone:
f.cache = NewNopCache()
return nil
default:
return ErrInvalidCacheType
}
@ -271,7 +272,7 @@ func (f *Fragment) openCache() error {
// Unmarshal cache data.
var pb internal.Cache
if err := proto.Unmarshal(buf, &pb); err != nil {
f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
f.Logger.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
return nil
}
@ -296,13 +297,13 @@ func (f *Fragment) Close() error {
func (f *Fragment) close() error {
// Flush cache if closing gracefully.
if err := f.flushCache(); err != nil {
f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
return err
}
// Close underlying storage.
if err := f.closeStorage(); err != nil {
f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
f.Logger.Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
return err
}
@ -341,9 +342,6 @@ func (f *Fragment) closeStorage() error {
return nil
}
// logger returns a logger instance for the fragment.nt.
func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) }
// Row returns a row by ID.
func (f *Fragment) Row(rowID uint64) *Bitmap {
f.mu.Lock()
@ -616,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e
return sum, count, nil
}
// FieldMin returns the min of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
count = x.Count()
if count > 0 {
consider = x
} else {
min += (1 << ii)
if ii == 0 {
count = consider.Count()
}
}
}
return min, count, nil
}
// FieldMax returns the max of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
// If there are no columns to consider, return early.
if consider.Count() == 0 {
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
count = x.Count()
if count > 0 {
max += (1 << ii)
consider = x
} else if ii == 0 {
count = consider.Count()
}
}
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
switch op {
@ -1020,7 +1082,7 @@ type TopOptions struct {
// Checksum returns a checksum for the entire fragment.
// If two fragments have the same checksum then they have the same data.
func (f *Fragment) Checksum() []byte {
h := sha1.New()
h := xxhash.New()
for _, block := range f.Blocks() {
h.Write(block.Checksum)
}
@ -1383,18 +1445,17 @@ func (f *Fragment) Snapshot() error {
defer f.mu.Unlock()
return f.snapshot()
}
func track(start time.Time, message string, stats StatsClient, logger *log.Logger) {
func track(start time.Time, message string, stats StatsClient, logger Logger) {
elapsed := time.Since(start)
logger.Printf("%s took %s", message, elapsed)
stats.Histogram("snapshot", elapsed.Seconds(), 1.0)
}
func (f *Fragment) snapshot() error {
logger := f.logger()
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
start := time.Now()
defer track(start, completeMessage, f.stats, logger)
defer track(start, completeMessage, f.stats, f.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
@ -1454,6 +1515,10 @@ func (f *Fragment) flushCache() error {
return nil
}
if f.CacheType == CacheTypeNone {
return nil
}
// Retrieve a list of row ids from the cache.
ids := f.cache.IDs()
@ -1660,7 +1725,7 @@ type blockHasher struct {
func newBlockHasher() blockHasher {
return blockHasher{
blockID: -1,
hash: sha1.New(),
hash: xxhash.New(),
}
}
func (h *blockHasher) Reset() {
@ -1680,7 +1745,7 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Host string
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -1701,7 +1766,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) SyncFragment() error {
// Determine replica set.
nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice())
nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice())
if len(nodes) == 1 {
return nil
}
@ -1710,17 +1775,14 @@ func (s *FragmentSyncer) SyncFragment() error {
blockSets := make([][]FragmentBlock, 0, len(nodes))
for _, node := range nodes {
// Read local blocks.
if node.Host == s.Host {
if node.ID == s.Node.ID {
b := s.Fragment.Blocks()
blockSets = append(blockSets, b)
continue
}
// Retrieve remote blocks.
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return err
@ -1786,8 +1848,8 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var pairSets []PairSet
var clients []InternalClient
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
if s.Host == node.Host {
for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) {
if s.Node.ID == node.ID {
continue
}
@ -1796,10 +1858,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
return nil
}
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
clients = append(clients, client)
// Only sync the standard block.
@ -1837,15 +1896,19 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.ColumnIDs) + len(clear.ColumnIDs)
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest))))
maxWrites := s.Cluster.MaxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
}
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
count++
}
@ -1861,7 +1924,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
Query: buffers[k].String(),
Remote: true,
}
_, err := clients[i].ExecuteQuery(context.Background(), f.Index(), queryRequest)
_, err := clients[i].Query(context.Background(), f.Index(), queryRequest)
if err != nil {
return err
}

99
fragment_test.go
View file

@ -163,7 +163,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
t.Fatal("expected change")
}
// Non-existant value.
// Non-existent value.
if value, exists, err := f.FieldValue(100, 11); err != nil {
t.Fatal(err)
} else if value != 0 {
@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) {
})
}
// Ensure a fragment can find the min and max of field values.
func TestFragment_FieldMinMax(t *testing.T) {
const bitDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil {
t.Fatal(err)
}
t.Run("Min", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 0, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if min != test.exp {
t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
t.Run("Max", func(t *testing.T) {
tests := []struct {
filter *pilosa.Bitmap
exp uint64
cnt uint64
}{
{filter: nil, exp: 2818, cnt: 2},
{filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1},
{filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2},
{filter: pilosa.NewBitmap(1), exp: 0, cnt: 0},
{filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1},
{filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil {
t.Fatal(err)
} else if max != test.exp {
t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max)
} else if cnt != test.cnt {
t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt)
}
}
})
}
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16
@ -702,7 +775,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
Fragment: frag,
RowAttrStore: test.MustOpenAttrStore(),
}
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
panic(err)
}
@ -1075,25 +1148,3 @@ func TestFragment_Snapshot_Run(t *testing.T) {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
func BenchmarkFragment_Snapshot(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
}
b.ReportAllocs()
// Open the fragment specified by the path.
f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Close()
b.ResetTimer()
// Reset timer and execute benchmark.
for i := 0; i < b.N; i++ {
err := f.Snapshot()
if err != nil {
b.Fatalf("unexpected count (reopen): %s", err)
}
}
}

430
frame.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -32,10 +31,8 @@ import (
// Default frame settings.
const (
DefaultRowLabel = "rowID"
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultRangeEnabled = false
// Default ranked frame cache
DefaultCacheSize = 50000
@ -43,31 +40,27 @@ const (
// Frame represents a container for views.
type Frame struct {
mu sync.RWMutex
path string
index string
name string
timeQuantum TimeQuantum
schema *FrameSchema
mu sync.RWMutex
path string
index string
name string
views map[string]*View
// Row attribute storage and cache
rowAttrStore *AttrStore
rowAttrStore AttrStore
broadcaster Broadcaster
Stats StatsClient
// Frame settings.
rowLabel string
cacheType string
// Frame options.
inverseEnabled bool
rangeEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
// Cache size for ranked frames
cacheSize uint32
LogOutput io.Writer
Logger Logger
}
// NewFrame returns a new instance of frame.
@ -78,24 +71,24 @@ func NewFrame(path, index, name string) (*Frame, error) {
}
return &Frame{
path: path,
index: index,
name: name,
schema: &FrameSchema{},
path: path,
index: index,
name: name,
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
views: make(map[string]*View),
rowAttrStore: NopAttrStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
rowLabel: DefaultRowLabel,
inverseEnabled: DefaultInverseEnabled,
rangeEnabled: DefaultRangeEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
//fields
LogOutput: ioutil.Discard,
Logger: NopLogger,
}, nil
}
@ -109,7 +102,7 @@ func (f *Frame) Index() string { return f.index }
func (f *Frame) Path() string { return f.path }
// RowAttrStore returns the attribute storage.
func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
@ -139,39 +132,6 @@ func (f *Frame) MaxInverseSlice() uint64 {
return view.MaxSlice()
}
// SetRowLabel sets the row labels. Persists to meta file on update.
func (f *Frame) SetRowLabel(v string) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ignore if no change occurred.
if v == "" || f.rowLabel == v {
return nil
}
// Make sure rowLabel is valid name
err := ValidateLabel(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
f.rowLabel = v
if err := f.saveMeta(); err != nil {
return err
}
return nil
}
// RowLabel returns the row label.
func (f *Frame) RowLabel() string {
f.mu.RLock()
v := f.rowLabel
f.mu.RUnlock()
return v
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
@ -182,11 +142,6 @@ func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
}
// RangeEnabled returns true if range fields can be stored on this frame.
func (f *Frame) RangeEnabled() bool {
return f.rangeEnabled
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Frame) SetCacheSize(v uint32) error {
@ -224,13 +179,11 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
RowLabel: f.rowLabel,
InverseEnabled: f.inverseEnabled,
RangeEnabled: f.rangeEnabled,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.schema.Fields,
Fields: f.fields,
}
}
@ -244,8 +197,6 @@ func (f *Frame) Open() error {
if err := f.loadMeta(); err != nil {
return err
} else if err := f.loadSchema(); err != nil {
return err
}
if err := f.openViews(); err != nil {
@ -304,12 +255,11 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.timeQuantum = ""
f.rowLabel = DefaultRowLabel
f.cacheType = DefaultCacheType
f.inverseEnabled = DefaultInverseEnabled
f.rangeEnabled = DefaultRangeEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
//f.fields
return nil
} else if err != nil {
return err
@ -320,17 +270,14 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rowLabel = pb.RowLabel
f.inverseEnabled = pb.InverseEnabled
f.rangeEnabled = pb.RangeEnabled
f.cacheSize = pb.CacheSize
// Copy cache type.
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
}
f.cacheSize = pb.CacheSize
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.fields = decodeFields(pb.Fields)
return nil
}
@ -352,35 +299,6 @@ func (f *Frame) saveMeta() error {
return nil
}
// loadSchema reads the schema for the frame.
func (f *Frame) loadSchema() error {
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema"))
if os.IsNotExist(err) {
f.schema = &FrameSchema{}
return nil
} else if err != nil {
return err
}
var pb internal.FrameSchema
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
f.schema = decodeFrameSchema(&pb)
return nil
}
// saveSchema writes the current schema to disk.
func (f *Frame) saveSchema() error {
if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil {
return err
}
return nil
}
// Close closes the frame and its views.
func (f *Frame) Close() error {
f.mu.Lock()
@ -402,16 +320,11 @@ func (f *Frame) Close() error {
return nil
}
// Schema returns the frame's current schema.
func (f *Frame) Schema() *FrameSchema {
// Field returns a field by name.
func (f *Frame) Field(name string) *Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.schema
}
// Field returns a field from the schema by name.
func (f *Frame) Field(name string) *Field {
for _, field := range f.Schema().Fields {
for _, field := range f.fields {
if field.Name == name {
return field
}
@ -419,42 +332,66 @@ func (f *Frame) Field(name string) *Field {
return nil
}
// CreateField creates a new field on the schema.
// Fields returns the fields on the frame.
func (f *Frame) Fields() []*Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.fields
}
// HasField returns true if a field exists on the frame.
func (f *Frame) HasField(name string) bool {
for _, fld := range f.fields {
if fld.Name == name {
return true
}
}
return false
}
// CreateField creates a new field on the frame.
func (f *Frame) CreateField(field *Field) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ensure frame supports fields.
if !f.RangeEnabled() {
return ErrFrameFieldsNotAllowed
}
// Copy schema and append field.
schema := f.schema.Clone()
if err := schema.AddField(field); err != nil {
// Append field.
if err := f.addField(field); err != nil {
return err
}
f.schema = schema
f.saveSchema()
f.saveMeta()
return nil
}
// addField adds a single field to fields.
func (f *Frame) addField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if f.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
f.fields = append(f.fields, field)
// Sort fields by name.
sort.Slice(f.fields, func(i, j int) bool {
return f.fields[i].Name < f.fields[j].Name
})
return nil
}
// GetFields returns a list of all the fields in the frame.
func (f *Frame) GetFields() (*FrameSchema, error) {
func (f *Frame) GetFields() ([]*Field, error) {
f.mu.RLock()
defer f.mu.RUnlock()
// Ensure the frame supports fields.
if !f.RangeEnabled() {
return nil, ErrFrameFieldsNotAllowed
}
err := f.loadSchema()
err := f.loadMeta()
if err != nil {
return nil, err
}
return f.schema, nil
return f.fields, nil
}
// DeleteField deletes an existing field on the schema.
@ -462,17 +399,10 @@ func (f *Frame) DeleteField(name string) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ensure frame supports fields.
if !f.RangeEnabled() {
return ErrFrameFieldsNotAllowed
}
// Copy schema and remove field.
schema := f.schema.Clone()
if err := schema.DeleteField(name); err != nil {
// Remove field.
if err := f.deleteField(name); err != nil {
return err
}
f.schema = schema
// Remove views.
viewName := ViewFieldPrefix + name
@ -489,6 +419,18 @@ func (f *Frame) DeleteField(name string) error {
return nil
}
// deleteField removes a single field from fields.
func (f *Frame) deleteField(name string) error {
for i, field := range f.fields {
if field.Name == name {
copy(f.fields[i:], f.fields[i+1:])
f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
f.mu.Lock()
@ -543,6 +485,18 @@ func (f *Frame) Views() []*View {
return other
}
// viewNames returns a list of all views (as a string) in the frame.
func (f *Frame) viewNames() []string {
f.mu.Lock()
defer f.mu.Unlock()
other := make([]string, 0, len(f.views))
for viewName, _ := range f.views {
other = append(other, viewName)
}
return other
}
// RecalculateCaches recalculates caches on every view in the frame.
func (f *Frame) RecalculateCaches() {
for _, view := range f.Views() {
@ -551,33 +505,60 @@ func (f *Frame) RecalculateCaches() {
}
// CreateViewIfNotExists returns the named view, creating it if necessary.
// Additionally, a CreateViewMessage is sent to the cluster.
func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
view, created, err := f.createViewIfNotExistsBase(name)
if err != nil {
return nil, err
}
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&internal.CreateViewMessage{
Index: f.index,
Frame: f.name,
View: name,
})
if err != nil {
return nil, err
}
}
return view, nil
}
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, ErrFrameInverseDisabled
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
if view := f.views[name]; view != nil {
return view, nil
return view, false, nil
}
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return nil, err
return nil, false, err
}
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
return view, nil
return view, true, nil
}
func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.index, f.name, name, f.cacheSize)
view.cacheType = f.cacheType
view.LogOutput = f.LogOutput
view.Logger = f.Logger
view.RowAttrStore = f.rowAttrStore
view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name))
view.broadcaster = f.broadcaster
@ -756,6 +737,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err
return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
}
// FieldMin returns the min for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmin, vcount, err := view.FieldMin(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmin) + field.Min, int64(vcount), nil
}
// FieldMax returns the max for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vmax, vcount, err := view.FieldMax(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vmax) + field.Min, int64(vcount), nil
}
func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)
@ -813,7 +834,11 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range rowIDs {
rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i]
rowID, columnID := rowIDs[i], columnIDs[i]
var timestamp *time.Time
if len(timestamps) > i {
timestamp = timestamps[i]
}
var standard, inverse []string
if timestamp == nil {
@ -883,11 +908,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// ImportValue bulk imports range-encoded value data.
func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
// Verify that this frame is range-encoded.
if !f.RangeEnabled() {
return fmt.Errorf("Frame not RangeEnabled: %s", f.name)
}
viewName := ViewFieldPrefix + fieldName
// Get the field so we know bitDepth.
field := f.Field(fieldName)
@ -956,8 +976,9 @@ func encodeFrames(a []*Frame) []*internal.Frame {
func encodeFrame(f *Frame) *internal.Frame {
fo := f.options()
return &internal.Frame{
Name: f.name,
Meta: fo.Encode(),
Name: f.name,
Meta: fo.Encode(),
Views: f.viewNames(),
}
}
@ -969,8 +990,9 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FrameInfo represents schema information for a frame.
type FrameInfo struct {
Name string `json:"name"`
Views []*ViewInfo `json:"views,omitempty"`
Name string `json:"name"`
Options FrameOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type frameInfoSlice []*FrameInfo
@ -981,9 +1003,8 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
RowLabel string `json:"rowLabel,omitempty"`
InverseEnabled bool `json:"inverseEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
@ -992,10 +1013,15 @@ type FrameOptions struct {
// Encode converts o into its internal representation.
func (o *FrameOptions) Encode() *internal.FrameMeta {
return encodeFrameOptions(o)
}
func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
if o == nil {
return nil
}
return &internal.FrameMeta{
RowLabel: o.RowLabel,
InverseEnabled: o.InverseEnabled,
RangeEnabled: o.RangeEnabled,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
@ -1003,74 +1029,16 @@ func (o *FrameOptions) Encode() *internal.FrameMeta {
}
}
// FrameSchema represents the list of fields on a frame.
type FrameSchema struct {
Fields []*Field
}
// Clone returns a copy of s.
func (s *FrameSchema) Clone() *FrameSchema {
other := &FrameSchema{Fields: make([]*Field, len(s.Fields))}
copy(other.Fields, s.Fields)
return other
}
// HasField returns true if a field exists on the schema.
func (s *FrameSchema) HasField(name string) bool {
for _, f := range s.Fields {
if f.Name == name {
return true
}
}
return false
}
// AddField adds a single field to the schema.
func (s *FrameSchema) AddField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if s.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
s.Fields = append(s.Fields, field)
// Sort fields by name.
sort.Slice(s.Fields, func(i, j int) bool {
return s.Fields[i].Name < s.Fields[j].Name
})
return nil
}
// DeleteField removes a single field from the schema.
func (s *FrameSchema) DeleteField(name string) error {
for i, field := range s.Fields {
if field.Name == name {
copy(s.Fields[i:], s.Fields[i+1:])
s.Fields, s.Fields[len(s.Fields)-1] = s.Fields[:len(s.Fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema {
if schema == nil {
func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
if options == nil {
return nil
}
return &internal.FrameSchema{
Fields: encodeFields(schema.Fields),
}
}
func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema {
if schema == nil {
return nil
}
return &FrameSchema{
Fields: decodeFields(schema.Fields),
return &FrameOptions{
InverseEnabled: options.InverseEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

48
frame_test.go
View file

@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) {
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
@ -267,49 +262,6 @@ func TestFrame_NameValidation(t *testing.T) {
}
}
// Ensure that frame RowLable validation is consistent.
func TestFrame_RowLabelValidation(t *testing.T) {
validRowLabels := []string{
"",
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
"camelCase",
"UPPERCASE",
}
invalidRowLabels := []string{
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
f, err := pilosa.NewFrame(path, "i", "f")
if err != nil {
t.Fatalf("unexpected frame error: %s", err)
}
for _, label := range validRowLabels {
if err := f.SetRowLabel(label); err != nil {
t.Fatalf("unexpected row label: %s %s", label, err)
}
}
for _, label := range invalidRowLabels {
if err := f.SetRowLabel(label); err == nil {
t.Fatalf("expected error on row label: %s", label)
}
}
}
// Ensure frame can open and retrieve a view.
func TestFrame_DeleteView(t *testing.T) {
f := test.MustOpenFrame()

41
gc.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// Ensure nopGCNotifier implements interface.
var _ GCNotifier = &nopGCNotifier{}
// GCNotifier represents an interface for garbage collection notificationss.
type GCNotifier interface {
Close()
AfterGC() <-chan struct{}
}
func init() {
NopGCNotifier = &nopGCNotifier{}
}
// NopGCNotifier represents a GCNotifier that doesn't do anything.
var NopGCNotifier GCNotifier
type nopGCNotifier struct{}
// Close is a no-op implementation of GCNotifier Close method.
func (n *nopGCNotifier) Close() {}
// AfterGC is a no-op implementation of GCNotifier AfterGC method.
func (c *nopGCNotifier) AfterGC() <-chan struct{} {
return nil
}

44
gcnotify/gcnotify.go Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gcnotify
import (
"github.com/CAFxX/gcnotifier"
"github.com/pilosa/pilosa"
)
// Ensure ActiveGCNotifier implements interface.
var _ pilosa.GCNotifier = &ActiveGCNotifier{}
type ActiveGCNotifier struct {
gcn *gcnotifier.GCNotifier
}
// NewActiveGCNotifier creates an active GCNotifier.
func NewActiveGCNotifier() *ActiveGCNotifier {
return &ActiveGCNotifier{
gcn: gcnotifier.New(),
}
}
// Close implements the GCNotifier interface.
func (n *ActiveGCNotifier) Close() {
n.gcn.Close()
}
// AfterGC implements the GCNotifier interface.
func (n *ActiveGCNotifier) AfterGC() <-chan struct{} {
return n.gcn.AfterGC()
}

115
gopsutil/systeminfo.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil
import (
"github.com/pilosa/pilosa"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
)
var _ pilosa.SystemInfo = NewSystemInfo()
// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS.
type SystemInfo struct {
platform string
family string
osVersion string
}
// Uptime returns the system uptime in seconds.
func (s *SystemInfo) Uptime() (uptime uint64, err error) {
hostInfo, err := host.Info()
if err != nil {
return 0, err
}
return hostInfo.Uptime, nil
}
// collectPlatformInfo fetches and caches system platform information.
func (s *SystemInfo) collectPlatformInfo() error {
var err error
if s.platform == "" {
s.platform, s.family, s.osVersion, err = host.PlatformInformation()
if err != nil {
return err
}
}
return nil
}
// Platform returns the system platform.
func (s *SystemInfo) Platform() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.platform, nil
}
// Family returns the system family.
func (s *SystemInfo) Family() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.family, err
}
// OSVersion returns the OS Version.
func (s *SystemInfo) OSVersion() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.osVersion, err
}
// MemFree returns the amount of free memory in bytes.
func (s *SystemInfo) MemFree() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Free, err
}
// MemTotal returns the amount of total memory in bytes.
func (s *SystemInfo) MemTotal() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Total, err
}
// MemUsed returns the amount of used memory in bytes.
func (s *SystemInfo) MemUsed() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Used, err
}
// KernelVersion returns the kernel version as a string.
func (s *SystemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *SystemInfo {
return &SystemInfo{}
}

77
gopsutil/systeminfo_test.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil_test
import (
"log"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gopsutil"
)
func TestSystemInfo(t *testing.T) {
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
// Uptime()(uint64, error)
// Platform()(string, error)
// Family()(string, error)
// OSVersion()(string, error)
// KernelVersion()(string, error)
// MemFree()(uint64, error)
// MemTotal()(uint64, error)
// MemUsed()(uint64, error)
//
uptime, err := systemInfo.Uptime()
if err != nil || uptime == 0 {
t.Fatalf("Error collecting uptime (error: %v)", err)
}
platform, err := systemInfo.Platform()
if err != nil {
t.Fatalf("Error getting platform. (platform: %v, error: %v)", platform, err)
}
family, err := systemInfo.Family()
if err != nil {
t.Fatalf("Error getting OS family. (family: %v, error: %v)", family, err)
}
osversion, err := systemInfo.OSVersion()
if err != nil {
t.Fatalf("Error getting OS version. (osversion: %v, error: %v)", osversion, err)
}
kernelversion, err := systemInfo.KernelVersion()
if err != nil {
t.Fatalf("Error getting kernel version. (kernelversion: %v, error: %v)", kernelversion, err)
}
memfree, err := systemInfo.MemFree()
if err != nil {
t.Fatalf("Error getting memfree. (memfree: %v, error: %v)", memfree, err)
}
memused, err := systemInfo.MemUsed()
if err != nil {
t.Fatalf("Error getting memused. (memused: %v, error: %v)", memused, err)
}
memtotal, err := systemInfo.MemTotal()
log.Println(memtotal)
if err != nil {
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
}
}

546
gossip/gossip.go
View file

@ -16,28 +16,32 @@ package gossip
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
// Ensure GossipNodeSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipNodeSet{}
var _ pilosa.Gossiper = &GossipNodeSet{}
var _ memberlist.Delegate = &GossipNodeSet{}
// Ensure GossipMemberSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipMemberSet{}
var _ pilosa.Gossiper = &GossipMemberSet{}
var _ memberlist.Delegate = &GossipMemberSet{}
// GossipNodeSet represents a gossip implementation of NodeSet using memberlist
// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster
// GossipNodeSet also represents an implementation of memberlist.Delegate
type GossipNodeSet struct {
// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
type GossipMemberSet struct {
mu sync.RWMutex
node *pilosa.Node
memberlist *memberlist.Memberlist
handler pilosa.BroadcastHandler
@ -46,50 +50,66 @@ type GossipNodeSet struct {
statusHandler pilosa.StatusHandler
config *gossipConfig
// The writer for any logging.
LogOutput io.Writer
Logger pilosa.Logger
logger *log.Logger
transport *Transport
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (g *GossipNodeSet) Nodes() []*pilosa.Node {
a := make([]*pilosa.Node, 0, g.memberlist.NumMembers())
for _, n := range g.memberlist.Members() {
a = append(a, &pilosa.Node{Scheme: "gossip", Host: n.Name})
}
return a
}
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler
func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error {
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler.
func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error {
g.handler = h
return nil
}
// Seed returns the gossipSeed determined by the config.
func (g *GossipNodeSet) Seed() string {
return g.config.gossipSeed
// GetBindAddr returns the gossip bind address based on config and auto bind port.
// This method is currently only used in a test scenario where a second node needs
// the auto-bind address of the first node to use as its gossip seed.
func (g *GossipMemberSet) GetBindAddr() string {
return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)
}
// Open implements the NodeSet interface to start network activity.
func (g *GossipNodeSet) Open() error {
// Open implements the MemberSet interface to start network activity.
func (g *GossipMemberSet) Open(n *pilosa.Node) error {
if g.handler == nil {
return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()")
return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()")
}
ml, err := memberlist.Create(g.config.memberlistConfig)
g.node = n
err := error(nil)
g.mu.Lock()
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
if err != nil {
return errors.Wrap(err, "creating memberlist")
}
g.memberlist = ml
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
g.mu.RLock()
defer g.mu.RUnlock()
return g.memberlist.NumMembers()
},
RetransmitMult: 3,
}
// attach to gossip seed node
nodes := []*pilosa.Node{&pilosa.Node{Scheme: "gossip", Host: g.config.gossipSeed}} //TODO: support a list of seeds
err = g.joinWithRetry(pilosa.Nodes(nodes).Hosts())
var uris = make([]*pilosa.URI, len(g.config.gossipSeeds))
for i, addr := range g.config.gossipSeeds {
uris[i], err = pilosa.NewURIFromAddress(addr)
if err != nil {
return fmt.Errorf("new uri from address: %s", err)
}
}
var nodes = make([]*pilosa.Node, len(uris))
for i, uri := range uris {
nodes[i] = &pilosa.Node{URI: *uri}
}
g.mu.RLock()
err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings())
g.mu.RUnlock()
if err != nil {
return errors.Wrap(err, "joinWithRetry")
}
@ -97,7 +117,7 @@ func (g *GossipNodeSet) Open() error {
}
// joinWithRetry wraps the standard memberlist Join function in a retry.
func (g *GossipNodeSet) joinWithRetry(hosts []string) error {
func (g *GossipMemberSet) joinWithRetry(hosts []string) error {
err := retry(60, 2*time.Second, func() error {
_, err := g.memberlist.Join(hosts)
return err
@ -121,128 +141,137 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) {
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
// logger returns a logger for the GossipNodeSet.
func (g *GossipNodeSet) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
////////////////////////////////////////////////////////////////
type gossipConfig struct {
gossipSeed string
gossipSeeds []string
memberlistConfig *memberlist.Config
}
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
// This is useful for test cases where specifiying a port is not reasonable.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if conf.LogOutput != nil && conf.Logger != nil {
return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.")
}
// GossipMemberSetOption describes a functional option for GossipMemberSet.
type GossipMemberSetOption func(*GossipMemberSet) error
logDest := conf.LogOutput
if logDest == nil {
logDest = os.Stderr
// WithTransport is a functional option for providing a transport to NewGossipMemberSet.
func WithTransport(transport *Transport) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
g.transport = transport
return nil
}
logger := conf.Logger
if logger == nil {
logger = log.New(logDest, "", log.LstdFlags)
}
nc := &memberlist.NetTransportConfig{
BindAddrs: []string{conf.BindAddr},
BindPort: conf.BindPort,
Logger: logger,
}
// See comment below for details about the retry in here.
makeNetRetry := func(limit int) (*memberlist.NetTransport, error) {
var err error
for try := 0; try < limit; try++ {
var nt *memberlist.NetTransport
if nt, err = memberlist.NewNetTransport(nc); err == nil {
return nt, nil
}
if strings.Contains(err.Error(), "address already in use") {
logger.Printf("[DEBUG] Got bind error: %v", err)
continue
}
}
return nil, fmt.Errorf("failed to obtain an address: %v", err)
}
// The dynamic bind port operation is inherently racy because
// even though we are using the kernel to find a port for us, we
// are attempting to bind multiple protocols (and potentially
// multiple addresses) with the same port number. We build in a
// few retries here since this often gets transient errors in
// busy unit tests.
limit := 1
if conf.BindPort == 0 {
limit = 10
}
nt, err := makeNetRetry(limit)
if err != nil {
return nil, fmt.Errorf("Could not set up network transport: %v", err)
}
if conf.BindPort == 0 {
port := nt.GetAutoBindPort()
conf.BindPort = port
conf.AdvertisePort = port
logger.Printf("[DEBUG] Using dynamic bind port %d", port)
}
return nt, nil
}
// NewGossipNodeSet returns a new instance of GossipNodeSet.
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipNodeSet, error) {
g := &GossipNodeSet{
LogOutput: server.LogOutput,
// WithLogger is a functional option for providing a logger to NewGossipMemberSet.
func WithLogger(logger *log.Logger) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
g.logger = logger
return nil
}
}
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
g := &GossipMemberSet{
Logger: pilosa.NopLogger,
}
conf := memberlist.DefaultWANConfig()
conf.BindPort = gossipPort
conf.AdvertisePort = gossipPort
// options
for _, opt := range options {
if err := opt(g); err != nil {
return nil, err
}
}
if g.transport == nil {
port, err := strconv.Atoi(cfg.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
// Set up the transport.
transport, err := NewTransport(host, port, g.logger)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
g.transport = transport
}
port := g.transport.Net.GetAutoBindPort()
var gossipKey []byte
var err error
if cfg.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Key)
if err != nil {
return nil, fmt.Errorf("reading gossip key: %s", err)
}
}
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = g.transport.Net
conf.Name = name
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.ProbeInterval)
conf.GossipNodes = cfg.Nodes
conf.GossipInterval = time.Duration(cfg.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime)
//
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = ger
conf.Logger = g.logger
//TODO: pull memberlist config from pilosa.cfg file
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeed: gossipSeed,
gossipSeeds: cfg.Seeds,
}
g.config.memberlistConfig.Name = name
g.config.memberlistConfig.BindAddr = gossipHost
g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost)
g.config.memberlistConfig.Delegate = g
g.config.memberlistConfig.SecretKey = secretKey
g.statusHandler = server
// set up the transport
transport, err := newTransport(g.config.memberlistConfig)
if err != nil {
return nil, err
}
g.config.memberlistConfig.Transport = transport
// If no gossipSeed is provided, use local host:port.
if gossipSeed == "" {
g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, g.config.memberlistConfig.BindPort)
}
g.statusHandler = sh
return g, nil
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipNodeSet) SendAsync(pb proto.Message) error {
// SendSync implementation of the Broadcaster interface.
func (g *GossipMemberSet) SendSync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return err
return fmt.Errorf("marshal message: %s", err)
}
mlist := g.memberlist
// Direct sends the message directly to every node.
// An error from any node raises an error on the entire operation.
//
// Gossip uses the gossip protocol to eventually deliver the message
// to every node.
var eg errgroup.Group
for _, n := range mlist.Members() {
// Don't send the message to the local node.
if n == mlist.LocalNode() {
continue
}
node := n
eg.Go(func() error {
return mlist.SendToTCP(node, msg)
})
}
return eg.Wait()
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
b := &broadcast{
@ -254,60 +283,136 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error {
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipNodeSet) NodeMeta(limit int) []byte {
return []byte{}
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
if err != nil {
g.Logger.Printf("marshal message error: %s", err)
return []byte{}
}
return buf
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipNodeSet) NotifyMsg(b []byte) {
func (g *GossipMemberSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
g.Logger.Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
g.Logger.Printf("receive message error: %s", err)
return
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte {
func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipNodeSet) LocalState(join bool) []byte {
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
g.Logger.Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
g.Logger.Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side side's LocalState.
func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) {
// receive and process the remote side's LocalState.
func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
g.Logger.Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
g.Logger.Printf("merge state error: %s", err)
}
}
// GossipEventReceiver is used to enable an application to receive
// events about joins and leaves over a channel.
//
// Care must be taken that events are processed in a timely manner from
// the channel, since this delegate will block until an event can be sent.
type GossipEventReceiver struct {
ch chan memberlist.NodeEvent
eventHandler pilosa.EventHandler
Logger pilosa.Logger
}
// NewGossipEventReceiver returns a new instance of GossipEventReceiver.
func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver {
return &GossipEventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
Logger: logger,
}
}
func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n}
}
func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n}
}
func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n}
}
// Start implements the pilosa.EventReceiver interface and sets the EventHandler.
func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error {
g.eventHandler = h
go g.listen()
return nil
}
func (g *GossipEventReceiver) listen() {
var nodeEventType pilosa.NodeEventType
for {
e := <-g.ch
switch e.Event {
case memberlist.NodeJoin:
nodeEventType = pilosa.NodeJoin
case memberlist.NodeLeave:
nodeEventType = pilosa.NodeLeave
case memberlist.NodeUpdate:
nodeEventType = pilosa.NodeUpdate
default:
continue
}
// Get the node from the event.Node meta data.
var n internal.Node
if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta data")
}
node := pilosa.DecodeNode(&n)
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: node,
}
if err := g.eventHandler.ReceiveEvent(ne); err != nil {
g.Logger.Printf("receive event error: %s", err)
}
}
}
@ -330,3 +435,148 @@ func (b *broadcast) Finished() {
close(b.notify)
}
}
// Transport is a gossip transport for binding to a port.
type Transport struct {
//memberlist.Transport
Net *memberlist.NetTransport
URI *pilosa.URI
}
// NewTransport returns a NetTransport based on the given host and port.
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.Logger = logger
net, err := newTransport(conf)
if err != nil {
return nil, fmt.Errorf("new transport: %s", err)
}
uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort()))
if err != nil {
return nil, fmt.Errorf("new uri from host port: %s", err)
}
return &Transport{
Net: net,
URI: uri,
}, nil
}
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
nc := &memberlist.NetTransportConfig{
BindAddrs: []string{conf.BindAddr},
BindPort: conf.BindPort,
Logger: conf.Logger,
}
// See comment below for details about the retry in here.
makeNetRetry := func(limit int) (*memberlist.NetTransport, error) {
var err error
for try := 0; try < limit; try++ {
var nt *memberlist.NetTransport
if nt, err = memberlist.NewNetTransport(nc); err == nil {
return nt, nil
}
if strings.Contains(err.Error(), "address already in use") {
conf.Logger.Printf("[DEBUG] Got bind error: %v", err)
continue
}
}
return nil, fmt.Errorf("failed to obtain an address: %v", err)
}
// The dynamic bind port operation is inherently racy because
// even though we are using the kernel to find a port for us, we
// are attempting to bind multiple protocols (and potentially
// multiple addresses) with the same port number. We build in a
// few retries here since this often gets transient errors in
// busy unit tests.
limit := 1
if conf.BindPort == 0 {
limit = 10
}
nt, err := makeNetRetry(limit)
if err != nil {
return nil, fmt.Errorf("Could not set up network transport: %v", err)
}
return nt, nil
}
// Config holds toml-friendly memberlist configuration.
type Config struct {
// Port indicates the port to which pilosa should bind for internal state sharing.
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
StreamTimeout toml.Duration `toml:"stream-timeout"`
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
SuspicionMult int `toml:"suspicion-mult"`
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
PushPullInterval toml.Duration `toml:"push-pull-interval"`
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
ProbeInterval toml.Duration `toml:"probe-interval"`
ProbeTimeout toml.Duration `toml:"probe-timeout"`
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
Interval toml.Duration `toml:"interval"`
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}

1372
handler.go
View file

File diff suppressed because it is too large Load diff

12
handler_internal_test.go
View file

@ -30,8 +30,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}},
{json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
}
for _, test := range tests {
actual := &postIndexRequest{}
@ -66,11 +65,10 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}},
{json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"},
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}},
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}},
{json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {
actual := &postFrameRequest{}

334
handler_test.go
View file

@ -19,6 +19,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
@ -31,18 +32,19 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/test"
)
func TestHandlerPanics(t *testing.T) {
h := test.NewHandler()
buf := &bytes.Buffer{}
h.Handler.LogOutput = buf
bufLogger := test.NewBufferLogger()
h.Handler.Logger = bufLogger
w := httptest.NewRecorder()
// will panic since Handler has no Holder set up
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil))
bufbytes, err := ioutil.ReadAll(buf)
bufbytes, err := bufLogger.ReadAll()
if err != nil {
t.Fatalf("reading all logoutput: %v", err)
}
@ -64,8 +66,8 @@ func TestHandler_NotFound(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
@ -99,13 +101,14 @@ func TestHandler_Schema(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -137,20 +140,55 @@ func TestHandler_Status(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.StatusHandler = s
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateNormal)
h.API.StatusHandler = s
s.Handler = h
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" {
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
func TestHandler_Info(t *testing.T) {
s := test.NewServer()
defer s.Close()
h := test.NewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.NewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateResizing)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != http.StatusOK {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
} else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
}
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
@ -165,13 +203,13 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -206,13 +244,13 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -223,8 +261,8 @@ func TestHandler_Query_Args_URL(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
@ -251,8 +289,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
@ -291,8 +329,8 @@ func TestHandler_Query_Args_Err(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
@ -306,7 +344,7 @@ func TestHandler_Query_Params_Err(t *testing.T) {
test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
@ -318,8 +356,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
@ -339,8 +377,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
@ -357,7 +395,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 100 {
t.Fatalf("unexpected n: %d", n)
}
@ -369,8 +407,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
@ -402,8 +440,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
@ -425,8 +463,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
@ -445,7 +483,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 {
@ -473,8 +511,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
@ -506,7 +544,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
@ -534,8 +572,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
@ -558,8 +596,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
@ -579,7 +617,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
@ -591,15 +629,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusInternalServerError {
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -612,8 +650,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
@ -622,7 +660,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
@ -640,8 +678,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != http.StatusMethodNotAllowed {
@ -655,8 +693,8 @@ func TestHandler_Query_ErrParse(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
@ -672,7 +710,7 @@ func TestHandler_Index_Delete(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
@ -712,8 +750,8 @@ func TestHandler_DeleteFrame(t *testing.T) {
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
@ -725,57 +763,13 @@ func TestHandler_DeleteFrame(t *testing.T) {
}
}
// Ensure handler can set the Index time quantum.
func TestHandler_SetIndexTimeQuantum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
// Ensure handler can set the frame time quantum.
func TestHandler_SetFrameTimeQuantum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
@ -824,7 +818,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
@ -874,12 +868,12 @@ func TestHandler_Frame_AddField(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -904,7 +898,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrInvalidFieldType", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -926,7 +920,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil {
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -949,8 +943,7 @@ func TestHandler_Frame_AddField(t *testing.T) {
t.Run("ErrFieldAlreadyExists", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
}); err != nil {
t.Fatal(err)
}
@ -978,12 +971,12 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@ -1011,7 +1004,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) {
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
@ -1043,21 +1036,18 @@ func TestHandler_Frame_GetFields(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil {
t.Fatal(err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f/fields")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
@ -1085,7 +1075,7 @@ func TestHandler_Frame_GetFields(t *testing.T) {
t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{})
resp, err := http.Get(s.URL + "/index/i/frame/f1/fields")
if err != nil {
@ -1093,12 +1083,12 @@ func TestHandler_Frame_GetFields(t *testing.T) {
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusBadRequest {
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) != `frame fields not allowed` {
t.Fatalf("unexpected body: %q", body)
} else if strings.TrimSpace(string(body)) == `frame fields not allowed` {
t.Fatalf("shouldn't get frame fields not allowed error: %q", body)
}
})
@ -1114,7 +1104,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
defer hldr.Close()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set bits in the index.
@ -1163,8 +1153,8 @@ func TestHandler_Version(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
@ -1186,17 +1176,33 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(3)
h.Cluster.ReplicaN = 2
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(3)
h.API.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
@ -1206,8 +1212,8 @@ func TestHandler_Expvars(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
@ -1252,8 +1258,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) {
]
}`)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusOK {
@ -1287,8 +1293,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
//Ensure throwing error if there's duplicated primaryKey field
invalidPrimaryKey := []byte(`
@ -1322,7 +1328,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) {
}
// Ensure throwing error if there's no primary key
hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"})
hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
unmatchColumnBody := []byte(`
{
"frames":[{
@ -1392,8 +1398,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Test index not found.
w := httptest.NewRecorder()
@ -1406,7 +1412,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
// Test input definition is deleted.
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -1440,10 +1446,10 @@ func TestHandler_GetInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -1607,8 +1613,8 @@ func TestHandler_CreateInput(t *testing.T) {
"null_value": null
}]`)
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Return error if index does not exist.
w := httptest.NewRecorder()
@ -1722,8 +1728,8 @@ func TestInput_JSON(t *testing.T) {
err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"},
}
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
for _, req := range tests {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json))))
@ -1783,8 +1789,8 @@ func TestHandler_DeleteView(t *testing.T) {
hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD")
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader("")))
if w.Code != http.StatusOK {
@ -1809,8 +1815,8 @@ func TestHandler_RecalculateCaches(t *testing.T) {
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
@ -1819,3 +1825,31 @@ func TestHandler_RecalculateCaches(t *testing.T) {
}
}
func TestHandler_WebUI(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
if !strings.Contains(w.Body.String(), "<title>Pilosa WebUI</title>") {
t.Fatalf("WebUI is not being served correctly.")
}
// If curl is the client, the response should be different
w = httptest.NewRecorder()
req := test.MustNewHTTPRequest("GET", "/", nil)
req.Header.Add("User-Agent", "curl/7.54.0")
h.ServeHTTP(w, req)
if !strings.Contains(w.Body.String(), "try the WebUI") {
t.Fatalf("WebUI is not being served correctly.")
}
}

321
holder.go
View file

@ -16,11 +16,8 @@ package pilosa
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
@ -31,6 +28,8 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -49,7 +48,13 @@ type Holder struct {
// Indexes by name.
indexes map[string]*Index
// opened channel is closed once Open() completes.
opened chan struct{}
Broadcaster Broadcaster
NewAttrStore func(string) AttrStore
// Close management
wg sync.WaitGroup
closing chan struct{}
@ -63,9 +68,7 @@ type Holder struct {
// The interval at which the cached row ids are persisted to disk.
CacheFlushInterval time.Duration
LogOutput io.Writer
LocalID string
Logger Logger
}
// NewHolder returns a new instance of Holder.
@ -74,12 +77,16 @@ func NewHolder() *Holder {
indexes: make(map[string]*Index),
closing: make(chan struct{}, 0),
opened: make(chan struct{}),
Broadcaster: NopBroadcaster,
Stats: NopStatsClient,
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
LogOutput: os.Stderr,
Logger: NopLogger,
}
}
@ -87,20 +94,21 @@ func NewHolder() *Holder {
func (h *Holder) Open() error {
h.setFileLimit()
h.Logger.Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return err
return errors.Wrap(err, "creating directory")
}
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
return err
return errors.Wrap(err, "opening directory")
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return err
return errors.Wrap(err, "reading directory")
}
for _, fi := range fis {
@ -108,30 +116,35 @@ func (h *Holder) Open() error {
continue
}
h.logger().Printf("opening index: %s", filepath.Base(fi.Name()))
h.Logger.Printf("opening index: %s", filepath.Base(fi.Name()))
index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err == ErrName {
h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
h.Logger.Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
continue
} else if err != nil {
return err
return errors.Wrap(err, "opening index")
}
if err := index.Open(); err != nil {
if err == ErrName {
h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err)
h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
continue
}
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
}
h.mu.Lock()
h.indexes[index.Name()] = index
h.mu.Unlock()
}
h.Logger.Printf("open holder: complete")
// Periodically flush cache.
h.wg.Add(1)
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
h.Stats.Open()
close(h.opened)
return nil
}
@ -145,12 +158,48 @@ func (h *Holder) Close() error {
for _, index := range h.indexes {
if err := index.Close(); err != nil {
return err
return errors.Wrap(err, "closing index")
}
}
return nil
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
func (h *Holder) HasData() (bool, error) {
h.mu.Lock()
defer h.mu.Unlock()
if len(h.indexes) > 0 {
return true, nil
}
// Open path to read all index directories.
if _, err := os.Stat(h.Path); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, errors.Wrap(err, "statting data dir")
}
f, err := os.Open(h.Path)
if err != nil {
return false, errors.Wrap(err, "opening data dir")
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return false, errors.Wrap(err, "reading data dir")
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
return true, nil
}
return false, nil
}
// MaxSlices returns MaxSlice map for all indexes.
func (h *Holder) MaxSlices() map[string]uint64 {
a := make(map[string]uint64)
@ -169,13 +218,13 @@ func (h *Holder) MaxInverseSlices() map[string]uint64 {
return a
}
// Schema returns schema data for all indexes and frames.
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name()}
fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
}
@ -189,6 +238,50 @@ func (h *Holder) Schema() []*IndexInfo {
return a
}
// ApplySchema applies an internal Schema to Holder.
func (h *Holder) ApplySchema(schema *internal.Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
opt := IndexOptions{}
idx, err := h.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return errors.Wrap(err, "creating index")
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
if err != nil {
return errors.Wrap(err, "creating frame")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
if err != nil {
return errors.Wrap(err, "creating view")
}
}
}
// TODO: Create inputDefinitions that don't exist.
}
return nil
}
// EncodeMaxSlices creates and internal representation of max slices.
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}
// EncodeSchema creates an internal representation of schema.
func (h *Holder) EncodeSchema() *internal.Schema {
return &internal.Schema{
Indexes: EncodeIndexes(h.Indexes()),
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -254,16 +347,14 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
// Otherwise create a new index.
index, err := h.newIndex(h.IndexPath(name), name)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "creating")
}
if err := index.Open(); err != nil {
return nil, err
return nil, errors.Wrap(err, "opening")
}
// Update options.
index.SetColumnLabel(opt.ColumnLabel)
index.SetTimeQuantum(opt.TimeQuantum)
h.indexes[index.Name()] = index
@ -275,9 +366,11 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
if err != nil {
return nil, err
}
index.LogOutput = h.LogOutput
index.Logger = h.Logger
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.Broadcaster
index.NewAttrStore = h.NewAttrStore
index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data"))
return index, nil
}
@ -294,12 +387,12 @@ func (h *Holder) DeleteIndex(name string) error {
// Close index.
if err := index.Close(); err != nil {
return err
return errors.Wrap(err, "closing")
}
// Delete index directory.
if err := os.RemoveAll(h.IndexPath(name)); err != nil {
return err
return errors.Wrap(err, "removing directory")
}
// Remove reference.
@ -363,7 +456,7 @@ func (h *Holder) flushCaches() {
}
if err := fragment.FlushCache(); err != nil {
h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
}
}
}
@ -387,7 +480,7 @@ func (h *Holder) setFileLimit() {
newLimit := &syscall.Rlimit{}
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
h.Logger.Printf("ERROR checking open file limit: %s", err)
return
}
// If the soft limit is lower than the FileLimit constant, we will try to change it.
@ -411,42 +504,68 @@ func (h *Holder) setFileLimit() {
}
// Try setting again with lowered Max (hard limit)
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil {
h.logger().Printf("ERROR setting open file limit: %s", err)
h.Logger.Printf("ERROR setting open file limit: %s", err)
}
// If we weren't trying to change the hard limit, let the user know something is wrong.
} else {
h.logger().Printf("ERROR setting open file limit: %s", err)
h.Logger.Printf("ERROR setting open file limit: %s", err)
}
}
// Check the limit after setting it. OS may not obey Setrlimit call.
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
h.Logger.Printf("ERROR checking open file limit: %s", err)
} else {
if oldLimit.Cur < FileLimit {
h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
}
}
}
}
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
func (h *Holder) loadLocalID() error {
idPath := path.Join(h.Path, "ID")
localID := ""
localIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
localID = strings.TrimSpace(string(localIDBytes))
} else {
u := uuid.NewV4()
localID = u.String()
err = ioutil.WriteFile(idPath, []byte(localID), 0600)
if err != nil {
return err
}
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, ".id")
nodeID := ""
h.Logger.Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return "", errors.Wrap(err, "creating directory")
}
h.LocalID = localID
nodeIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
nodeID = strings.TrimSpace(string(nodeIDBytes))
} else if os.IsNotExist(err) {
nodeID = uuid.NewV4().String()
err = ioutil.WriteFile(idPath, []byte(nodeID), 0600)
if err != nil {
return "", errors.Wrap(err, "writing file")
}
} else if err != nil {
return "", errors.Wrap(err, "reading file")
}
return nodeID, nil
}
// Log startup time and version to $DATA_DIR/.startup.log
func (h *Holder) logStartup() error {
time, err := time.Now().MarshalText()
if err != nil {
return errors.Wrap(err, "creating timestamp")
}
logLine := fmt.Sprintf("%s\t%s\n", time, Version)
f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return errors.Wrap(err, "opening startup log")
}
defer f.Close()
if _, err = f.WriteString(logLine); err != nil {
return errors.Wrap(err, "writing startup log")
}
return nil
}
@ -455,7 +574,7 @@ func (h *Holder) loadLocalID() error {
type HolderSyncer struct {
Holder *Holder
URI *URI
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -511,7 +630,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.URI.HostPort(), di.Name, slice) {
if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
continue
}
@ -548,36 +667,33 @@ func (s *HolderSyncer) syncIndex(index string) error {
// Read block checksums.
blks, err := idx.ColumnAttrStore().Blocks()
if err != nil {
return err
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.ColumnAttrDiff(context.Background(), index, blks)
if err != nil {
return err
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.Host})
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID})
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
return err
return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = idx.ColumnAttrStore().Blocks()
if err != nil {
return err
return errors.Wrap(err, "recomputing blocks")
}
}
@ -586,7 +702,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve index reference.
// Retrieve frame reference.
f := s.Holder.Frame(index, name)
if f == nil {
return nil
@ -597,16 +713,13 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return err
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
@ -614,21 +727,21 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
} else if err != nil {
return err
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.Host})
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
return err
return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = f.RowAttrStore().Blocks()
if err != nil {
return err
return errors.Wrap(err, "recomputing blocks")
}
}
@ -646,26 +759,88 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
// Ensure view exists locally.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
return err
return errors.Wrap(err, "creating view")
}
// Ensure fragment exists locally.
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return err
return errors.Wrap(err, "creating fragment")
}
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Host: s.URI.HostPort(),
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
}
if err := fs.SyncFragment(); err != nil {
return err
return errors.Wrap(err, "syncing fragment")
}
return nil
}
// HolderCleaner removes fragments and data files that are no longer used.
type HolderCleaner struct {
Node *Node
Holder *Holder
Cluster *Cluster
// Signals that the sync should stop.
Closing <-chan struct{}
}
// IsClosing returns true if the cleaner has been marked to close.
func (c *HolderCleaner) IsClosing() bool {
select {
case <-c.Closing:
return true
default:
return false
}
}
// CleanHolder compares the holder with the cluster state and removes
// any unnecessary fragments and files.
func (c *HolderCleaner) CleanHolder() error {
for _, index := range c.Holder.Indexes() {
// Verify cleaner has not closed.
if c.IsClosing() {
return nil
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
continue
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
return errors.Wrap(err, "deleting fragment")
}
}
}
}
}
return nil
}
func uint64InSlice(i uint64, s []uint64) bool {
for _, o := range s {
if i == o {
return true
}
}
return false
}

240
holder_test.go
View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"bytes"
"context"
"os"
"path/filepath"
@ -24,12 +25,17 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexName", func(t *testing.T) {
h := test.MustOpenHolder()
bufLogger := test.NewBufferLogger()
h.Holder.Logger = bufLogger
defer h.Close()
if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil {
@ -39,8 +45,12 @@ func TestHolder_Open(t *testing.T) {
}
if err := h.Reopen(); err != nil {
t.Fatal(err)
} else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) {
t.Fatalf("expected log error:\n%s", logOutput)
}
if bufbytes, err := bufLogger.ReadAll(); err != nil {
t.Fatal(err)
} else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) {
t.Fatalf("expected log error:\n%s", bufbytes)
}
})
@ -64,22 +74,6 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrIndexMetaCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
@ -284,6 +278,53 @@ func TestHolder_Open(t *testing.T) {
})
}
func TestHolder_HasData(t *testing.T) {
t.Run("IndexDirectory", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if ok, err := h.HasData(); !ok || err != nil {
t.Fatal("expected HasData to return true, but ", ok, err)
}
})
t.Run("Peek", func(t *testing.T) {
h := test.NewHolder()
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
// Create an index directory to indicate data exists.
if err := os.Mkdir(h.IndexPath("test"), 0777); err != nil {
t.Fatal(err)
}
if ok, err := h.HasData(); !ok || err != nil {
t.Fatal("expected HasData to return true, no err, but", ok, err)
}
})
t.Run("Peek at missing directory", func(t *testing.T) {
h := test.NewHolder()
// Ensure that hasData is false when dir doesn't exist.
h.Path = "bad-path"
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)
}
})
}
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
@ -320,7 +361,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
cluster := test.NewCluster(2)
client := pilosa.GetHTTPClient(nil)
client := server.GetHTTPClient(nil)
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
@ -330,20 +371,25 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
defer hldr1.Close()
s := test.NewServer()
defer s.Close()
s.Handler.Holder = hldr1.Holder
s.Handler.API.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(client)
e.Holder = hldr1.Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].Host = "localhost:0"
cluster.Nodes[1].Host = test.MustParseURLHost(s.URL)
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
t.Fatal(err)
}
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0, hldr1} {
@ -395,15 +441,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr0.Index("y").SetRemoteMaxSlice(3)
// Set up syncer.
uri, err := cluster.Nodes[0].URI()
if err != nil {
t.Fatal(err)
}
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
URI: uri,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: pilosa.GetHTTPClient(nil),
RemoteClient: server.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
}
@ -440,3 +482,141 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
}
}
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
t.Fatal(err)
}
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(2)
// Keep replication the same and ensure we get the expected results.
cluster.ReplicaN = 2
// Set up cleaner for replication 2.
cleaner2 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner2.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
// Change replication factor to ensure we have fragments to remove.
cluster.ReplicaN = 1
// Set up cleaner for replication 1.
cleaner1 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner1.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
if f != nil {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
}

236
index.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -31,7 +30,6 @@ import (
// Default index settings.
const (
DefaultColumnLabel = "columnID"
InputDefinitionDir = ".input-definitions"
)
@ -41,13 +39,6 @@ type Index struct {
path string
name string
// Default time quantum for all frames in index.
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Label used for referring to columns in index.
columnLabel string
// Frames by name.
frames map[string]*Frame
@ -55,8 +46,10 @@ type Index struct {
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
NewAttrStore func(string) AttrStore
// Column attribute storage and cache.
columnAttrStore *AttrStore
columnAttrStore AttrStore
// InputDefinitions by name.
inputDefinitions map[string]*InputDefinition
@ -64,7 +57,7 @@ type Index struct {
broadcaster Broadcaster
Stats StatsClient
LogOutput io.Writer
Logger Logger
}
// NewIndex returns a new instance of Index.
@ -83,13 +76,12 @@ func NewIndex(path, name string) (*Index, error) {
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
columnLabel: DefaultColumnLabel,
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
LogOutput: ioutil.Discard,
Logger: NopLogger,
}, nil
}
@ -100,40 +92,7 @@ func (i *Index) Name() string { return i.name }
func (i *Index) Path() string { return i.path }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (i *Index) SetColumnLabel(v string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if no change occurred.
if v == "" || i.columnLabel == v {
return nil
}
// Make sure columnLabel is valid name
err := ValidateLabel(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
i.columnLabel = v
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// ColumnLabel returns the column label.
func (i *Index) ColumnLabel() string {
i.mu.RLock()
v := i.columnLabel
i.mu.RUnlock()
return v
}
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
// Options returns all options for this index.
func (i *Index) Options() IndexOptions {
@ -143,10 +102,7 @@ func (i *Index) Options() IndexOptions {
}
func (i *Index) options() IndexOptions {
return IndexOptions{
ColumnLabel: i.columnLabel,
TimeQuantum: i.timeQuantum,
}
return IndexOptions{}
}
// Open opens and initializes the index.
@ -213,8 +169,6 @@ func (i *Index) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
if os.IsNotExist(err) {
i.timeQuantum = ""
i.columnLabel = DefaultColumnLabel
return nil
} else if err != nil {
return err
@ -225,19 +179,18 @@ func (i *Index) loadMeta() error {
}
// Copy metadata fields.
i.timeQuantum = TimeQuantum(pb.TimeQuantum)
i.columnLabel = pb.ColumnLabel
return nil
}
// NOTE: Until we introduce new attributes to store in the index .meta file,
// we don't need to actually write the file. The code related to index.options
// and the index meta file are left in place for future use.
/*
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{
TimeQuantum: string(i.timeQuantum),
ColumnLabel: i.columnLabel,
})
buf, err := proto.Marshal(&internal.IndexMeta{})
if err != nil {
return err
}
@ -249,6 +202,7 @@ func (i *Index) saveMeta() error {
return nil
}
*/
// Close closes the index and its frames.
func (i *Index) Close() error {
@ -256,9 +210,7 @@ func (i *Index) Close() error {
defer i.mu.Unlock()
// Close the attribute store.
if i.columnAttrStore != nil {
i.columnAttrStore.Close()
}
i.columnAttrStore.Close()
// Close all frames.
for _, f := range i.frames {
@ -321,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.remoteMaxInverseSlice = v
}
// TimeQuantum returns the default time quantum for the index.
func (i *Index) TimeQuantum() TimeQuantum {
i.mu.RLock()
defer i.mu.RUnlock()
return i.timeQuantum
}
// SetTimeQuantum sets the default time quantum for the index.
func (i *Index) SetTimeQuantum(q TimeQuantum) error {
i.mu.Lock()
defer i.mu.Unlock()
// Validate input.
if !q.Valid() {
return ErrInvalidTimeQuantum
}
// Update value on index.
i.timeQuantum = q
// Perist meta data to disk.
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
@ -392,6 +316,20 @@ func (i *Index) Frames() []*Frame {
return a
}
// InputDefinitions returns a list of all inputDefinitions in the index.
func (i *Index) InputDefinitions() []*InputDefinition {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*InputDefinition, 0, len(i.inputDefinitions))
for _, d := range i.inputDefinitions {
a = append(a, d)
}
//sort.Sort(inputDefintionSlice(a)) // TODO
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
@ -431,22 +369,9 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, ErrInvalidCacheType
}
// Validate that row label does not match column label.
if i.columnLabel == opt.RowLabel || (opt.RowLabel == "" && i.columnLabel == DefaultRowLabel) {
return nil, ErrColumnRowLabelEqual
}
// Validate mutually exclusive options if ranges are enabled.
if opt.RangeEnabled {
if opt.InverseEnabled {
return nil, ErrInverseRangeNotAllowed
} else if opt.CacheType != "" && opt.CacheType != CacheTypeNone {
return nil, ErrRangeCacheNotAllowed
}
} else {
if len(opt.Fields) > 0 {
return nil, ErrFrameFieldsNotAllowed
}
i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame")
}
// Validate fields.
@ -467,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, err
}
// Default the time quantum to what is set on the Index.
timeQuantum := i.timeQuantum
if opt.TimeQuantum != "" {
timeQuantum = opt.TimeQuantum
}
if err := f.SetTimeQuantum(timeQuantum); err != nil {
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
return nil, err
}
@ -483,33 +404,20 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
}
f.cacheType = opt.CacheType
// Set options.
if opt.RowLabel != "" {
f.rowLabel = opt.RowLabel
}
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
f.rangeEnabled = opt.RangeEnabled
// Set fields.
f.fields = opt.Fields
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
f.rangeEnabled = opt.RangeEnabled
// Set schema & save.
f.schema = &FrameSchema{
Fields: opt.Fields,
}
if err := f.saveSchema(); err != nil {
f.Close()
return nil, err
}
// Add to index's frame lookup.
i.frames[name] = f
@ -521,9 +429,10 @@ func (i *Index) newFrame(path, name string) (*Frame, error) {
if err != nil {
return nil, err
}
f.LogOutput = i.LogOutput
f.Logger = i.Logger
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
return f, nil
}
@ -572,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexInfoSlice) Len() int { return len(p) }
func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// MergeSchemas combines indexes and frames from a and b into one schema.
func MergeSchemas(a, b []*IndexInfo) []*IndexInfo {
// Generate a map from both schemas.
m := make(map[string]map[string]map[string]struct{})
for _, idxs := range [][]*IndexInfo{a, b} {
for _, idx := range idxs {
if m[idx.Name] == nil {
m[idx.Name] = make(map[string]map[string]struct{})
}
for _, frame := range idx.Frames {
if m[idx.Name][frame.Name] == nil {
m[idx.Name][frame.Name] = make(map[string]struct{})
}
for _, view := range frame.Views {
m[idx.Name][frame.Name][view.Name] = struct{}{}
}
}
}
}
// Generate new schema from map.
idxs := make([]*IndexInfo, 0, len(m))
for idx, frames := range m {
di := &IndexInfo{Name: idx}
for frame, views := range frames {
fi := &FrameInfo{Name: frame}
for view := range views {
fi.Views = append(fi.Views, &ViewInfo{Name: view})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
idxs = append(idxs, di)
}
sort.Sort(indexInfoSlice(idxs))
return idxs
}
// EncodeIndexes converts a into its internal representation.
func EncodeIndexes(a []*Index) []*internal.Index {
other := make([]*internal.Index, len(a))
@ -623,27 +492,19 @@ func EncodeIndexes(a []*Index) []*internal.Index {
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
io := d.options()
return &internal.Index{
Name: d.name,
Meta: io.Encode(),
MaxSlice: d.MaxSlice(),
Frames: encodeFrames(d.Frames()),
Name: d.name,
Frames: encodeFrames(d.Frames()),
InputDefinitions: encodeInputDefinitions(d.InputDefinitions()),
}
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
type IndexOptions struct{}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
ColumnLabel: i.ColumnLabel,
TimeQuantum: string(i.TimeQuantum),
}
return &internal.IndexMeta{}
}
// hasTime returns true if a contains a non-nil time.
@ -688,7 +549,6 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin
for _, fr := range pb.Frames {
opt := FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
RowLabel: DefaultRowLabel,
InverseEnabled: fr.Meta.InverseEnabled,
CacheType: fr.Meta.CacheType,
CacheSize: fr.Meta.CacheSize,
@ -723,7 +583,6 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) {
if err != nil {
return nil, err
}
inputDef.broadcaster = i.broadcaster
return inputDef, nil
}
@ -776,7 +635,6 @@ func (i *Index) openInputDefinitions() error {
return nil
}
}
}
return nil
}
@ -784,7 +642,8 @@ func (i *Index) openInputDefinitions() error {
// InputBits Process the []Bit though the Frame import process
func (i *Index) InputBits(frame string, bits []*Bit) error {
var rowIDs, columnIDs []uint64
timestamps := make([]*time.Time, len(bits))
var timestamps []*time.Time
f := i.Frame(frame)
if f == nil {
return fmt.Errorf("Frame not found: %s", frame)
@ -799,6 +658,11 @@ func (i *Index) InputBits(frame string, bits []*Bit) error {
// Convert timestamps to time.Time.
if bit.Timestamp > 0 {
// Don't create a full timestamps slice unless
// at least one bit contains a timestamp.
if len(timestamps) == 0 {
timestamps = make([]*time.Time, len(bits))
}
t := time.Unix(bit.Timestamp, 0)
timestamps[i] = &t
}

165
index_test.go
View file

@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame with explicit quantum.
f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
if err != nil {
@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) {
t.Fatalf("unexpected frame time quantum: %s", q)
}
})
t.Run("Inherited", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame.
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
t.Fatalf("unexpected frame time quantum: %s", q)
}
})
})
// Ensure frame can include range columns.
@ -99,55 +76,81 @@ func TestIndex_CreateFrame(t *testing.T) {
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
RangeEnabled: false,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema: %#v", f.Schema())
t.Fatalf("unexpected fields: %#v", f.Fields())
}
// Reopen the index & verify the fields are loaded.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema after reopen: %#v", f.Schema())
t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
}
})
t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) {
t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
InverseEnabled: true,
frame, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
}); err != pilosa.ErrInverseRangeNotAllowed {
InverseEnabled: true,
Fields: []*pilosa.Field{
&pilosa.Field{
Name: "myfield",
Type: pilosa.FieldTypeInt,
Min: -20,
Max: 100,
},
},
})
if err != nil {
t.Fatal(err)
}
ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetFieldValue(1, "myfield", 87)
if !ch || err != nil {
t.Fatal(ch, err)
}
views := frame.Views()
if len(views) != 3 {
var names string
for _, v := range views {
names = names + v.Name() + " "
}
t.Fatalf("Unexpected views: %s", names)
}
})
t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) {
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeRanked,
}); err != pilosa.ErrRangeCacheNotAllowed {
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
})
@ -156,15 +159,14 @@ func TestIndex_CreateFrame(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) {
t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
@ -172,7 +174,7 @@ func TestIndex_CreateFrame(t *testing.T) {
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFrameFieldsNotAllowed {
}); err != nil {
t.Fatal(err)
}
})
@ -182,7 +184,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
@ -196,7 +197,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
@ -210,7 +210,7 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
@ -219,32 +219,6 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
})
// Ensure frame cannot be created with a matching row label.
t.Run("ErrColumnRowLabelEqual", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
_, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel})
if err != pilosa.ErrColumnRowLabelEqual {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("Default", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil {
t.Fatal(err)
}
_, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != pilosa.ErrColumnRowLabelEqual {
t.Fatalf("unexpected error: %s", err)
}
})
})
}
// Ensure index can delete a frame.
@ -270,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) {
}
}
// Ensure index can set the default time quantum.
func TestIndex_SetTimeQuantum(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set & retrieve time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload index and verify that it is persisted.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure index can delete a frame.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
@ -307,7 +261,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}}
@ -334,7 +288,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) {
}
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -354,7 +308,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -385,7 +339,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -407,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
err := index.InputBits("f", bits)
if !strings.Contains(err.Error(), "Frame not found") {
t.Fatalf("Expected Frame not found error, actual error: %s", err)
}
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil {
t.Fatal(err)
}

62
input_definition.go
View file

@ -36,12 +36,11 @@ var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleR
// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
name string
path string
index string
broadcaster Broadcaster
frames []InputFrame
fields []InputDefinitionField
name string
path string
index string
frames []InputFrame
fields []InputDefinitionField
}
// NewInputDefinition returns a new instance of InputDefinition.
@ -86,17 +85,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
// Copy metadata fields.
i.name = pb.Name
for _, fr := range pb.Frames {
frameMeta := fr.Meta
inputFrame := InputFrame{
Name: fr.Name,
Options: FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
RowLabel: DefaultRowLabel,
InverseEnabled: frameMeta.InverseEnabled,
CacheSize: frameMeta.CacheSize,
CacheType: frameMeta.CacheType,
TimeQuantum: TimeQuantum(frameMeta.TimeQuantum),
},
Name: fr.Name,
Options: *decodeFrameOptions(fr.Meta),
}
i.frames = append(i.frames, inputFrame)
}
@ -290,7 +281,7 @@ func (i *InputDefinitionInfo) Validate() error {
}
}
// Validate columnLabel and duplicate primaryKey.
// Validate duplicate primaryKey.
for _, field := range i.Fields {
if field.Name == "" {
return ErrInputDefinitionNameRequired
@ -338,6 +329,43 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition {
return &def
}
// encodeInputDefinitions converts a into its internal representation.
func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition {
other := make([]*internal.InputDefinition, len(a))
for i := range a {
other[i] = encodeInputDefinition(a[i])
}
return other
}
// encodeInputDefinition converts i into its internal representation.
func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition {
//fo := f.options()
return &internal.InputDefinition{
Name: i.name,
Frames: encodeInputFrames(i.frames),
Fields: encodeInputDefinitionFields(i.fields),
}
}
// encodeInputFrames converts a into its internal representation.
func encodeInputFrames(a []InputFrame) []*internal.Frame {
other := make([]*internal.Frame, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// encodeInputDefinitionFields converts a into its internal representation.
func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField {
other := make([]*internal.InputDefinitionField, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// AddFrame manually add frame to input definition.
func (i *InputDefinition) AddFrame(frame InputFrame) error {
i.frames = append(i.frames, frame)

8
input_definition_test.go
View file

@ -30,7 +30,7 @@ func TestInputDefinition_Open(t *testing.T) {
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
@ -115,14 +115,14 @@ func TestActionValidation(t *testing.T) {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err)
}
frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if !strings.Contains(err.Error(), "rowID required for single-row-boolean") {
t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err)
}
frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
@ -131,7 +131,7 @@ func TestActionValidation(t *testing.T) {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err)
}
frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}

4318
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

109
internal/private.proto
View file

@ -3,12 +3,9 @@ syntax = "proto3";
package internal;
message IndexMeta {
string ColumnLabel = 1;
string TimeQuantum = 2;
}
message FrameMeta {
string RowLabel = 1;
bool InverseEnabled = 2;
string CacheType = 3;
uint32 CacheSize = 4;
@ -38,8 +35,9 @@ message Cache {
repeated uint64 IDs = 1;
}
message MaxSlicesResponse {
map<string, uint64> MaxSlices = 1;
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
@ -68,17 +66,31 @@ message DeleteFrameMessage {
string Frame = 2;
}
message CreateFieldMessage {
string Index = 1;
string Frame = 2;
Field Field = 3;
}
message DeleteFieldMessage {
string Index = 1;
string Frame = 2;
string Field = 3;
}
message Frame {
string Name = 1;
FrameMeta Meta = 2;
repeated string Views = 3;
}
message Schema {
repeated Index Indexes = 1;
}
message Index {
string Name = 1;
IndexMeta Meta = 2;
uint64 MaxSlice = 3;
repeated Frame Frames = 4;
repeated uint64 Slices = 5;
repeated InputDefinition InputDefinitions = 6;
}
@ -112,19 +124,38 @@ message DeleteInputDefinitionMessage {
string Name = 2;
}
message NodeStatus {
string Host = 1;
message URI {
string Scheme = 1;
string Host = 2;
uint32 Port = 3;
}
message Node {
string ID = 1;
URI URI = 2;
bool IsCoordinator = 3;
}
message NodeStateMessage {
string NodeID = 1;
string State = 2;
repeated Index Indexes = 3;
string Scheme = 4;
}
message NodeEventMessage {
uint32 Event = 1;
Node Node = 2;
}
message NodeStatus {
Node Node = 1;
MaxSlices MaxSlices = 2;
Schema Schema = 3;
}
message ClusterStatus {
repeated NodeStatus Nodes = 1;
}
message FrameSchema {
repeated Field Fields = 1;
string ClusterID = 1;
string State = 2;
repeated Node Nodes = 3;
}
message Field {
@ -134,8 +165,52 @@ message Field {
int64 Max = 4;
}
message CreateViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message ResizeInstruction {
int64 JobID = 1;
Node Node = 2;
Node Coordinator = 3;
repeated ResizeSource Sources = 4;
Schema Schema = 5;
ClusterStatus ClusterStatus = 6;
}
message ResizeSource {
Node Node = 1;
string Index = 2;
string Frame = 3;
string View = 4;
uint64 Slice = 5;
}
message ResizeInstructionComplete {
int64 JobID = 1;
Node Node = 2;
string Error = 3;
}
message SetCoordinatorMessage {
Node New = 1;
}
message UpdateCoordinatorMessage {
Node New = 1;
}
message Topology {
string ClusterID = 1;
repeated string NodeIDs = 2;
}
message RecalculateCaches {}

142
internal/public.pb.go
View file

@ -11,7 +11,7 @@
It has these top-level messages:
Bitmap
Pair
SumCount
ValCount
Bit
ColumnAttrSet
Attr
@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 {
return 0
}
type SumCount struct {
Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"`
type ValCount struct {
Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"`
Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
func (m *SumCount) Reset() { *m = SumCount{} }
func (m *SumCount) String() string { return proto.CompactTextString(m) }
func (*SumCount) ProtoMessage() {}
func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *ValCount) Reset() { *m = ValCount{} }
func (m *ValCount) String() string { return proto.CompactTextString(m) }
func (*ValCount) ProtoMessage() {}
func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (m *SumCount) GetSum() int64 {
func (m *ValCount) GetVal() int64 {
if m != nil {
return m.Sum
return m.Val
}
return 0
}
func (m *SumCount) GetCount() int64 {
func (m *ValCount) GetCount() int64 {
if m != nil {
return m.Count
}
@ -358,7 +358,7 @@ type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"`
ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
}
@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair {
return nil
}
func (m *QueryResult) GetSumCount() *SumCount {
func (m *QueryResult) GetValCount() *ValCount {
if m != nil {
return m.SumCount
return m.ValCount
}
return nil
}
@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 {
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*SumCount)(nil), "internal.SumCount")
proto.RegisterType((*ValCount)(nil), "internal.ValCount")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *SumCount) Marshal() (dAtA []byte, err error) {
func (m *ValCount) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *SumCount) MarshalTo(dAtA []byte) (int, error) {
func (m *ValCount) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Sum != 0 {
if m.Val != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Sum))
i = encodeVarintPublic(dAtA, i, uint64(m.Val))
}
if m.Count != 0 {
dAtA[i] = 0x10
@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.SumCount != nil {
if m.ValCount != nil {
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size()))
n6, err := m.SumCount.MarshalTo(dAtA[i:])
i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size()))
n6, err := m.ValCount.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) {
return n
}
func (m *SumCount) Size() (n int) {
func (m *ValCount) Size() (n int) {
var l int
_ = l
if m.Sum != 0 {
n += 1 + sovPublic(uint64(m.Sum))
if m.Val != 0 {
n += 1 + sovPublic(uint64(m.Val))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) {
if m.Changed {
n += 2
}
if m.SumCount != nil {
l = m.SumCount.Size()
if m.ValCount != nil {
l = m.ValCount.Size()
n += 1 + l + sovPublic(uint64(l))
}
if m.Type != 0 {
@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *SumCount) Unmarshal(dAtA []byte) error {
func (m *ValCount) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SumCount: wiretype end group for non-group")
return fmt.Errorf("proto: ValCount: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType)
}
m.Sum = 0
m.Val = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Sum |= (int64(b) & 0x7F) << shift
m.Val |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
m.Changed = bool(v != 0)
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.SumCount == nil {
m.SumCount = &SumCount{}
if m.ValCount == nil {
m.ValCount = &ValCount{}
}
if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{
// 705 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3,
0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3,
0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76,
0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce,
0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c,
0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85,
0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45,
0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a,
0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7,
0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7,
0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a,
0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9,
0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb,
0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21,
0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce,
0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42,
0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22,
0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15,
0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb,
0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b,
0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45,
0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec,
0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed,
0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10,
0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a,
0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94,
0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29,
0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb,
0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80,
0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd,
0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36,
0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc,
0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b,
0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4,
0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a,
0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad,
0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d,
0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2,
0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5,
0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23,
0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8,
0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b,
0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03,
0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad,
0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8,
0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f,
0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91,
0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc,
0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f,
0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4,
0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00,
0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d,
0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8,
0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7,
0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69,
0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55,
0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b,
0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b,
0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85,
0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a,
0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47,
0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0,
0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56,
0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06,
0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b,
0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0,
0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f,
0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23,
0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9,
0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde,
0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8,
0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00,
0x00,
}

6
internal/public.proto
View file

@ -14,8 +14,8 @@ message Pair {
uint64 Count = 2;
}
message SumCount {
int64 Sum = 1;
message ValCount {
int64 Val = 1;
int64 Count = 2;
}
@ -64,7 +64,7 @@ message QueryResult {
Bitmap Bitmap = 1;
uint64 N = 2;
repeated Pair Pairs = 3;
SumCount SumCount = 5;
ValCount ValCount = 5;
bool Changed = 4;
}

88
logger.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io"
"log"
)
// Ensure nopLogger implements interface.
var _ Logger = &nopLogger{}
// Logger represents an interface for a shared logger.
type Logger interface {
Printf(format string, v ...interface{})
Debugf(format string, v ...interface{})
}
func init() {
NopLogger = &nopLogger{}
}
// NopLogger represents a Logger that doesn't do anything.
var NopLogger Logger
type nopLogger struct{}
// Printf is a no-op implementation of the Logger Printf method.
func (n *nopLogger) Printf(format string, v ...interface{}) {}
// Debugf is a no-op implementation of the Logger Debugf method.
func (n *nopLogger) Debugf(format string, v ...interface{}) {}
// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger.
type StandardLogger struct {
logger *log.Logger
}
func NewStandardLogger(w io.Writer) *StandardLogger {
return &StandardLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (s *StandardLogger) Printf(format string, v ...interface{}) {
s.logger.Printf(format, v...)
}
func (s *StandardLogger) Debugf(format string, v ...interface{}) {}
func (s *StandardLogger) Logger() *log.Logger {
return s.logger
}
// VerboseLogger is an implementation of pilosa.Logger which includes debug messages.
type VerboseLogger struct {
logger *log.Logger
}
func NewVerboseLogger(w io.Writer) *VerboseLogger {
return &VerboseLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (vb *VerboseLogger) Printf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Debugf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Logger() *log.Logger {
return vb.logger
}

133
lru/lru.go Normal file
View file

@ -0,0 +1,133 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package lru implements an LRU cache.
package lru
import "container/list"
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
OnEvicted func(key Key, value interface{})
ll *list.List
cache map[interface{}]*list.Element
}
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
type Key interface{}
type entry struct {
key Key
value interface{}
}
// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
}
// Add adds a value to the cache.
func (c *Cache) Add(key Key, value interface{}) {
if c.cache == nil {
c.cache = make(map[interface{}]*list.Element)
c.ll = list.New()
}
if ee, ok := c.cache[key]; ok {
c.ll.MoveToFront(ee)
ee.Value.(*entry).value = value
return
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
}
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.ll.MoveToFront(ele)
return ele.Value.(*entry).value, true
}
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.removeElement(ele)
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
if c.cache == nil {
return
}
ele := c.ll.Back()
if ele != nil {
c.removeElement(ele)
}
}
func (c *Cache) removeElement(e *list.Element) {
c.ll.Remove(e)
kv := e.Value.(*entry)
delete(c.cache, kv.key)
if c.OnEvicted != nil {
c.OnEvicted(kv.key, kv.value)
}
}
// Len returns the number of items in the cache.
func (c *Cache) Len() int {
if c.cache == nil {
return 0
}
return c.ll.Len()
}
// Clear purges all stored items from the cache.
func (c *Cache) Clear() {
if c.OnEvicted != nil {
for _, e := range c.cache {
kv := e.Value.(*entry)
c.OnEvicted(kv.key, kv.value)
}
}
c.ll = nil
c.cache = nil
}

100
pilosa.go
View file

@ -36,31 +36,26 @@ var (
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal")
ErrInputDefinitionExists = errors.New("input-definition already exists")
ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey")
ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey")
ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel")
ErrInputDefinitionNameRequired = errors.New("input-definition name required")
ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required")
ErrInputDefinitionValueMap = errors.New("valueMap required for map")
ErrInputDefinitionActionRequired = errors.New("field definitions require an action")
ErrInputDefinitionNotFound = errors.New("input-definition not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
ErrRangeCacheNotAllowed = errors.New("range cache not allowed")
ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
@ -73,16 +68,29 @@ var (
ErrQueryRequired = errors.New("query required")
ErrTooManyWrites = errors.New("too many write commands")
ErrConfigClusterTypeInvalid = errors.New("invalid cluster type")
ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts")
ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
)
// ApiMethodNotAllowedError wraps an error value indicating that a particular
// API method is not allowed in the current cluster state.
type ApiMethodNotAllowedError struct {
error
}
// BadRequestError wraps an error value to signify that a request could not be
// read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest
// would be returned.
type BadRequestError struct {
error
}
// Regular expression to validate index and frame names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// Regular expression to validate row and column labels.
var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
@ -144,14 +152,6 @@ func ValidateName(name string) error {
return nil
}
// ValidateLabel ensures that the label is a valid format.
func ValidateLabel(label string) error {
if labelRegexp.Match([]byte(label)) == false {
return ErrLabel
}
return nil
}
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
@ -162,6 +162,50 @@ func StringInSlice(a string, list []string) bool {
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {

24
pilosa_test.go
View file

@ -46,30 +46,6 @@ func TestValidateNameInvalid(t *testing.T) {
}
}
func TestValidateLabel(t *testing.T) {
labels := []string{
"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, label := range labels {
if pilosa.ValidateLabel(label) != nil {
t.Fatalf("Should be valid label: %s", label)
}
}
}
func TestValidateLabelInvalid(t *testing.T) {
labels := []string{
"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, label := range labels {
if pilosa.ValidateLabel(label) == nil {
t.Fatalf("Should be invalid label: %s", label)
}
}
}
func TestStringInSlice(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "localhost:10101"

68
roaring/assembly.go
View file

@ -1,68 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
// bit population count, take from
// https://code.google.com/p/go/issues/detail?id=4988#c11
// credit: https://code.google.com/u/arnehormann/
func popcntGo(x uint64) (n uint64) {
x -= (x >> 1) & 0x5555555555555555
x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
x += x >> 4
x &= 0x0f0f0f0f0f0f0f0f
x *= 0x0101010101010101
return x >> 56
}
func popcntSliceGo(s []uint64) uint64 {
cnt := uint64(0)
for _, x := range s {
cnt += popcntGo(x)
}
return cnt
}
func popcntMaskSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] &^ m[i])
}
return cnt
}
func popcntAndSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] & m[i])
}
return cnt
}
func popcntOrSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] | m[i])
}
return cnt
}
func popcntXorSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] ^ m[i])
}
return cnt
}

122
roaring/assembly_amd64.s
View file

@ -1,122 +0,0 @@
#include "textflag.h"
TEXT ·hasAsm(SB),4,$0-1
MOVQ $1, AX
CPUID
SHRQ $23, CX
ANDQ $1, CX
MOVB CX, ret+0(FP)
RET
TEXT ·POPCNTQ(SB),NOSPLIT,$0-16
MOVQ memory+0(FP), BP
POPCNTQ BP, BX
MOVQ BX, ret+8(FP)
RET
TEXT ·BSFQ(SB),NOSPLIT,$0-16
MOVQ memory+0(FP), BP
BSFQ BP, BX
MOVQ BX, ret+8(FP)
RET
#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2
TEXT ·popcntSliceAsm(SB),4,$0-32
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntSliceEnd
popcntSliceLoop:
BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX
ADDQ DX, AX
ADDQ $8, SI
LOOP popcntSliceLoop
popcntSliceEnd:
MOVQ AX, ret+24(FP)
RET
TEXT ·popcntMaskSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntMaskSliceEnd
MOVQ m+24(FP), DI
popcntMaskSliceLoop:
MOVQ (DI), DX
NOTQ DX
ANDQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntMaskSliceLoop
popcntMaskSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntAndSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntAndSliceEnd
MOVQ m+24(FP), DI
popcntAndSliceLoop:
MOVQ (DI), DX
ANDQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntAndSliceLoop
popcntAndSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntOrSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntOrSliceEnd
MOVQ m+24(FP), DI
popcntOrSliceLoop:
MOVQ (DI), DX
ORQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntOrSliceLoop
popcntOrSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntXorSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntXorSliceEnd
MOVQ m+24(FP), DI
popcntXorSliceLoop:
MOVQ (DI), DX
XORQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntXorSliceLoop
popcntXorSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntAsm(SB),4,$0-16
MOVQ x+0(FP), DX
POPCNTQ_DX_DX
MOVQ DX, ret+8(FP)
RET

87
roaring/assembly_asm.go
View file

@ -1,87 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build amd64
package roaring
func hasAsm() bool
func BSFQ(memory uint64) int
func POPCNTQ(memory uint64) int
//go:noescape
var useAsm = hasAsm()
//go:noescape
func popcntSliceAsm(s []uint64) uint64
//go:noescape
func popcntMaskSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntAndSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntOrSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntXorSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntAsm(x uint64) uint64
func popcntSlice(s []uint64) uint64 {
if useAsm {
return popcntSliceAsm(s)
}
return popcntSliceGo(s)
}
func popcntMaskSlice(s, m []uint64) uint64 {
if useAsm {
return popcntMaskSliceAsm(s, m)
}
return popcntMaskSliceGo(s, m)
}
func popcntAndSlice(s, m []uint64) uint64 {
if useAsm {
return popcntAndSliceAsm(s, m)
}
return popcntAndSliceGo(s, m)
}
func popcntOrSlice(s, m []uint64) uint64 {
if useAsm {
return popcntOrSliceAsm(s, m)
}
return popcntOrSliceGo(s, m)
}
func popcntXorSlice(s, m []uint64) uint64 {
if useAsm {
return popcntXorSliceAsm(s, m)
}
return popcntXorSliceGo(s, m)
}
func popcnt(x uint64) uint64 {
if useAsm {
return popcntAsm(x)
}
return popcntGo(x)
}

Some files were not shown because too many files have changed in this diff Show more