scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge

Browse source
This commit is contained in:
Todd Gruben 2018-04-02 09:43:35 -05:00
commit 58ca5cf13a
95 changed files with 11766 additions and 3817 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.travis.yml
View file

@ -1,14 +1,14 @@
language: go
go:
- 1.8
- 1.9
- "1.10" # Use string, as 1.10==1.1 if interpreted as float.
- master
env:
global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
install:
- make vendor
- make vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
@ -19,10 +19,11 @@ before_deploy:
- pip install awscli --user `whoami`
deploy:
- provider: script
script: make prerelease-upload
script: make prerelease-upload GOOS=linux GOARCH=amd64
skip_cleanup: true
on:
branch: master
all_branches: true
go: 1.9
matrix:
allow_failures:
- go: master

106
CHANGELOG.md
View file

@ -5,6 +5,112 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
### Added
- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972))
### Changed
- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
### Fixed
- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
### Performance
- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
## [0.8.8] - 2018-02-19
This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions.
### Fixed
- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
## [0.8.7] - 2018-02-12
This version contains 1 contribution from 1 contributors. There are 2 files changed, 84 insertions, and 4 deletions.
### Fixed
- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
## [0.8.6] - 2018-02-09
This version contains 2 contributions from 2 contributors. There are 3 files changed, 171 insertions, and 6 deletions.
### Fixed
- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
## [0.8.5] - 2018-01-18
This version contains 1 contribution from 1 contributor. There is 1 file changed, 1 insertion, and 0 deletions.
### Fixed
- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
## [0.8.4] - 2018-01-10
This version contains 4 contributions from 3 contributors. There are 17 files changed, 974 insertions, and 221 deletions.
### Fixed
- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
### Performance
- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
## [0.8.3] - 2017-12-12
This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions.

77
CONTRIBUTING.md
View file

@ -2,30 +2,83 @@
## Reporting a bug
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
## Submitting a feature request
Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4]
Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
which starts by [opening a new issue][1] that describes the new feature proposal.
## Submitting code changes
## Making code contributions
Before you start working on new features, you should [open a new issue][1] to let others know what
you're doing before you start working, otherwise you run the risk of duplicating effort. This also
you're doing, otherwise you run the risk of duplicating effort. This also
gives others an opportunity to provide input for your feature.
If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
- Fork the [Pilosa repository][2] and then clone your fork:
```shell
git clone git@github.com:<your-name>/pilosa.git
### Development Environment
- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information.
- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
- Fork the [Pilosa repository][2] to your own account.
- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone git@github.com:${USER}/pilosa.git
```
- `cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
- [Install](https://github.com/golang/dep/#installation) `dep` to manage dependencies:
```sh
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
```
- Install Pilosa command line tools:
```sh
make install
```
or
```
dep ensure && go install github.com/pilosa/pilosa/cmd/...
```
Running `pilosa` should now run a Pilosa instance.
- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add upstream git@github.com:pilosa/pilosa.git
```
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:
```sh
git fetch upstream
git checkout master
git merge upstream/master
```
- Create a local feature branch:
```shell
```sh
git checkout -b something-amazing
```
@ -33,13 +86,13 @@ If you want to help but you aren't sure where to start, check out our [github la
- Make sure that you've written tests for your new feature, and then run the tests:
```shell
```sh
make test
```
- Verify that your pull request is applied to the latest version of code on github:
```shell
```sh
git remote add upstream git@github.com:pilosa/pilosa.git
git fetch upstream
git rebase -i upstream/master
@ -47,7 +100,7 @@ If you want to help but you aren't sure where to start, check out our [github la
- Push to your fork:
```shell
```sh
git push -u <yourfork> something-amazing
```
@ -59,4 +112,4 @@ If you want to help but you aren't sure where to start, check out our [github la
[3]: https://github.com/pilosa/pilosa/compare/
[4]: https://github.com/pilosa/general/blob/master/proposal.md
[5]: https://github.com/pilosa/pilosa/issues
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

2
Dockerfile
View file

@ -1,4 +1,4 @@
FROM golang:1.9.2 as builder
FROM golang:1.10 as builder
ARG ldflags=''

10
Gopkg.lock generated
View file

@ -88,8 +88,8 @@
[[projects]]
name = "github.com/gorilla/mux"
packages = ["."]
revision = "7f08801859139f86dfafd1c296e2cba9a80d292e"
version = "v1.6.0"
revision = "53c1911da2b537f792e7cafcb446b05ffe33b996"
version = "v1.6.1"
[[projects]]
branch = "master"
@ -205,12 +205,6 @@
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
name = "github.com/sony/gobreaker"
packages = ["."]
revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36"
version = "0.3.0"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"

17
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test
DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
@ -11,6 +11,9 @@ PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
BUILD_TIME=`date -u +%FT%T%z`
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
DOCKER_GOLANG_IMAGE=golang:latest
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH))
BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH)
default: test pilosa
@ -75,16 +78,16 @@ else
endif
prerelease-build: vendor
make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa"
cp LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH)
tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/
@echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz"
make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa"
cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/
@echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz"
prerelease:
make prerelease-build GOOS=linux GOARCH=amd64
prerelease-upload: prerelease
aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read
aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read
install: vendor
go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
@ -100,7 +103,7 @@ generate-protoc: .protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
generate-statik: statik
go generate github.com/pilosa/pilosa
go generate github.com/pilosa/pilosa/statik
generate: generate-protoc generate-statik

69
README-dev.md
View file

@ -1,69 +0,0 @@
Development Environment
=======================
Install Go versions 1.6.2+ or 1.7 for your platform.
Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private.
Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`.
Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
```sh
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone git@github.com:${USER}/pilosa.git
```
`cd` to your pilosa directory:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
```
Install `dep` to manage dependencies:
```sh
go get -u github.com/golang/dep/cmd/dep
```
Install Pilosa command line tools:
```sh
make install
# or:
# dep ensure && go install github.com/pilosa/pilosa/cmd/...
```
Running `pilosa` should now run a Pilosa instance.
In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
```sh
cd ${GOPATH}/src/github.com/pilosa/pilosa
git remote add upstream git@github.com:pilosa/pilosa.git
```
Before starting to work on a task, sync your branch with the upstream:
```sh
git fetch upstream
git checkout master
git merge upstream/master
```
Create a branch for the task:
```sh
git checkout -b a-branch-for-the-task
```
Update the code in the branch, and commit it.
Push it to your own repo:
```sh
git push --set-upstream origin a-branch-for-the-task
```
All left to do is creating a pull request on github.com.

513
attr.go
View file

@ -16,23 +16,12 @@ package pilosa
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// Attribute data type enum.
const (
AttrTypeString = 1
@ -41,313 +30,111 @@ const (
AttrTypeFloat = 4
)
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
// AttrStore represents an interface for handling row/column attributes.
type AttrStore interface {
Path() string
Open() error
Close() error
Attrs(id uint64) (m map[string]interface{}, err error)
SetAttrs(id uint64, m map[string]interface{}) error
SetBulkAttrs(m map[uint64]map[string]interface{}) error
Blocks() ([]AttrBlock, error)
BlockData(i uint64) (map[uint64]map[string]interface{}, error)
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
func init() {
NopAttrStore = &nopAttrStore{}
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
// NopAttrStore represents an AttrStore that doesn't do anything.
var NopAttrStore AttrStore
func NewNopAttrStore(string) AttrStore {
return &nopAttrStore{}
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// nopAttrStore represents a no-op implementation of the AttrStore interface.
type nopAttrStore struct{}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) *AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
// Path is a no-op implementation of AttrStore Path method.
func (s *nopAttrStore) Path() string { return "" }
// Open is a no-op implementation of AttrStore Open method.
func (s *nopAttrStore) Open() error {
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
// Close is a no-op implementation of AttrStore Close method.
func (s *nopAttrStore) Close() error {
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
// Attrs is a no-op implementation of AttrStore Attrs method.
func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return nil, nil
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
return nil
}
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
// Blocks is a no-op implementation of AttrStore Blocks method.
func (s *nopAttrStore) Blocks() ([]AttrBlock, error) {
return nil, nil
}
// BlockData is a no-op implementation of AttrStore BlockData method.
func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []AttrBlock
for cur.nextBlock() {
block := AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
m[btou64(k)] = decodeAttrs(pb.GetAttrs())
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
a, other = a[1:], other[1:]
}
}
// Marshal and save new values.
buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
func encodeAttrs(m map[string]interface{}) []*internal.Attr {
@ -421,136 +208,16 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} {
return other
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
// EncodeAttrs encodes an attribute map into a byte slice.
func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
a, other = a[1:], other[1:]
}
}
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
// DecodeAttrs decodes a byte slice into an attribute map.
func DecodeAttrs(v []byte) (map[string]interface{}, error) {
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}

2
bitmap.go
View file

@ -14,8 +14,6 @@
package pilosa
// #cgo CFLAGS:-mpopcnt
import (
"encoding/json"
"sort"

465
boltdb/attrstore.go Normal file
View file

@ -0,0 +1,465 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []pilosa.AttrBlock
for cur.nextBlock() {
block := pilosa.AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
return nil, err
}
m[btou64(k)] = attrs
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
return pilosa.DecodeAttrs(v)
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
}
}
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}

130
broadcast.go
View file

@ -22,38 +22,27 @@ import (
"github.com/pilosa/pilosa/internal"
)
// NodeSet represents an interface for Node membership and inter-node communication.
type NodeSet interface {
// Returns a list of all Nodes in the cluster
Nodes() []*Node
// Open starts any network activity implemented by the NodeSet
Open() error
// MemberSet represents an interface for Node membership and inter-node communication.
type MemberSet interface {
// Open starts any network activity implemented by the MemberSet
// Node is the local node, used for membership broadcasts.
Open(n *Node) error
}
// StaticNodeSet represents a basic NodeSet for testing.
type StaticNodeSet struct {
// StaticMemberSet represents a basic MemberSet for testing.
type StaticMemberSet struct {
nodes []*Node
}
// NewStaticNodeSet creates a statically defined NodeSet.
func NewStaticNodeSet() *StaticNodeSet {
return &StaticNodeSet{}
// NewStaticMemberSet creates a statically defined MemberSet.
func NewStaticMemberSet(nodes []*Node) *StaticMemberSet {
return &StaticMemberSet{
nodes: nodes,
}
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (s *StaticNodeSet) Nodes() []*Node {
return s.nodes
}
// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing.
func (s *StaticNodeSet) Open() error {
return nil
}
// Join sets the NodeSet nodes to the slice of Nodes passed in.
func (s *StaticNodeSet) Join(nodes []*Node) error {
s.nodes = nodes
// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing.
func (s *StaticMemberSet) Open(n *Node) error {
return nil
}
@ -61,6 +50,7 @@ func (s *StaticNodeSet) Join(nodes []*Node) error {
type Broadcaster interface {
SendSync(pb proto.Message) error
SendAsync(pb proto.Message) error
SendTo(to *Node, pb proto.Message) error
}
func init() {
@ -73,16 +63,21 @@ var NopBroadcaster Broadcaster
type nopBroadcaster struct{}
// SendSync A no-op implemenetation of Broadcaster SendSync method.
// SendSync A no-op implementation of Broadcaster SendSync method.
func (n *nopBroadcaster) SendSync(pb proto.Message) error {
return nil
}
// SendAsync A no-op implemenetation of Broadcaster SendAsync method.
// SendAsync A no-op implementation of Broadcaster SendAsync method.
func (n *nopBroadcaster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error {
return nil
}
// BroadcastHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type BroadcastHandler interface {
@ -117,21 +112,32 @@ var NopGossiper Gossiper
type nopGossiper struct{}
// SendAsync A no-op implemenetation of Gossiper SendAsync method.
// SendAsync A no-op implementation of Gossiper SendAsync method.
func (n *nopGossiper) SendAsync(pb proto.Message) error {
return nil
}
// Broadcast message types.
const (
MessageTypeCreateSlice = 1
MessageTypeCreateIndex = 2
MessageTypeDeleteIndex = 3
MessageTypeCreateFrame = 4
MessageTypeDeleteFrame = 5
MessageTypeCreateInputDefinition = 6
MessageTypeDeleteInputDefinition = 7
MessageTypeDeleteView = 8
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateFrame
MessageTypeDeleteFrame
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateInputDefinition
MessageTypeDeleteInputDefinition
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -148,12 +154,34 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
typ = MessageTypeDeleteFrame
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
case *internal.CreateInputDefinitionMessage:
typ = MessageTypeCreateInputDefinition
case *internal.DeleteInputDefinitionMessage:
typ = MessageTypeDeleteInputDefinition
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -180,12 +208,34 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:
m = &internal.DeleteFrameMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateInputDefinition:
m = &internal.CreateInputDefinitionMessage{}
case MessageTypeDeleteInputDefinition:
m = &internal.DeleteInputDefinitionMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
}

38
client.go
View file

@ -89,9 +89,6 @@ func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[st
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.clientURI(ctx), "/slices/max")
u.RawQuery = (&url.Values{
"inverse": {strconv.FormatBool(inverse)},
}).Encode()
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
@ -108,14 +105,17 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
}
defer resp.Body.Close()
var rsp sliceMaxResponse
var rsp getSlicesMaxResponse
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
return rsp.MaxSlices, nil
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
// Schema returns all index and frame schema information.
@ -296,7 +296,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
// Import to each node.
for _, node := range nodes {
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -317,13 +317,12 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, b
}
node := &Node{
Scheme: c.defaultURI.Scheme(),
Host: c.defaultURI.HostPort(),
URI: *c.defaultURI,
}
// Import to node.
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
return nil
@ -448,7 +447,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel
// Import to each node.
for _, node := range nodes {
if err := c.importValueNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -536,7 +535,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
return nil
@ -688,6 +687,13 @@ func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, frame, view, slice, node)
}
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
@ -717,7 +723,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
return nil, ErrFragmentNotFound
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.Host, resp.StatusCode)
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
return resp.Body, nil
@ -796,7 +802,7 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i
// Return error if response not OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.Host, resp.StatusCode)
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
}
@ -1130,6 +1136,8 @@ func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI {
clientURI := c.defaultURI
if contextURI, ok := ctx.Value("uri").(*URI); ok {
clientURI = contextURI
} else if contextURI, ok := ctx.Value("uri").(URI); ok {
clientURI = &contextURI
}
return clientURI
}
@ -1318,8 +1326,8 @@ func uriPathToURL(uri *URI, path string) url.URL {
func nodePathToURL(node *Node, path string) url.URL {
return url.URL{
Scheme: node.Scheme,
Host: node.Host,
Scheme: node.URI.Scheme(),
Host: node.URI.HostPort(),
Path: path,
}
}

43
client_test.go
View file

@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
for i := 0; i < numNodes; i++ {
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
server[i].Handler.URI = server[i].HostURI()
server[i].Handler.Cluster = c
server[i].Handler.Cluster.Nodes[i].Host = server[i].Host()
server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.Holder = hldr[i].Holder
server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i]
}
return server, hldr
}
@ -64,24 +64,21 @@ func TestClient_MultiNode(t *testing.T) {
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[0].Holder
e.Scheme = cluster.Nodes[0].Scheme
e.Host = cluster.Nodes[0].Host
e.Node = cluster.Nodes[0]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[1].Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[2].Holder
e.Scheme = cluster.Nodes[2].Scheme
e.Host = cluster.Nodes[2].Host
e.Node = cluster.Nodes[2]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
@ -89,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host())
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
if ownNum == num {
@ -220,10 +217,10 @@ func TestClient_Import(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -271,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -320,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -358,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -423,10 +420,10 @@ func TestClient_BackupInverseView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -460,10 +457,10 @@ func TestClient_BackupInvalidView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -489,10 +486,10 @@ func TestClient_FragmentBlocks(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)

1731
cluster.go
View file

File diff suppressed because it is too large Load diff

309
cluster_internal_test.go Normal file
View file

@ -0,0 +1,309 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io/ioutil"
"reflect"
"testing"
"github.com/pilosa/pilosa/internal"
)
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
c := NewCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
tests := []struct {
idx string
maxSlice uint64
frameViews viewsByFrame
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
},
},
{
idx: "foo",
maxSlice: uint64(3),
frameViews: viewsByFrame{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
},
},
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(name string) *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, name)
if err != nil {
panic(err)
}
return index
}
// Ensure that fragSources creates the correct fragment mapping.
func TestFragSources(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
uri3, err := NewURIFromAddress("host3")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node3 := &Node{ID: "node3", URI: *uri3}
c1 := NewCluster()
c1.ReplicaN = 1
c1.addNodeBasicSorted(node0)
c1.addNodeBasicSorted(node1)
c2 := NewCluster()
c2.ReplicaN = 1
c2.addNodeBasicSorted(node0)
c2.addNodeBasicSorted(node1)
c2.addNodeBasicSorted(node2)
c3 := NewCluster()
c3.ReplicaN = 2
c3.addNodeBasicSorted(node0)
c3.addNodeBasicSorted(node1)
c4 := NewCluster()
c4.ReplicaN = 2
c4.addNodeBasicSorted(node0)
c4.addNodeBasicSorted(node1)
c4.addNodeBasicSorted(node2)
c5 := NewCluster()
c5.ReplicaN = 2
c5.addNodeBasicSorted(node0)
c5.addNodeBasicSorted(node1)
c5.addNodeBasicSorted(node2)
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 1300000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 2600000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 3900000, nil)
if err != nil {
t.Fatal(err)
}
tests := []struct {
from *Cluster
to *Cluster
idx *Index
expected map[string][]*internal.ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{},
"node1": []*internal.ResizeSource{},
"node2": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c4,
to: c3,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c5,
to: c4,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
},
"node2": []*internal.ResizeSource{},
},
err: "",
},
{
from: c2,
to: c4,
idx: idx,
expected: nil,
err: "clusters are the same size",
},
{
from: c1,
to: c5,
idx: idx,
expected: nil,
err: "adding more than one node at a time is not supported",
},
{
from: c5,
to: c1,
idx: idx,
expected: nil,
err: "removing more than one node at a time is not supported",
},
}
for _, test := range tests {
actual, err := (test.from).fragSources(test.to, test.idx)
if test.err != "" {
if err.Error() != test.err {
t.Fatalf("expected error: %s", test.err)
}
} else {
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
}
// Ensure that fragSources creates the correct fragment mapping.
func TestResizeJob(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
tests := []struct {
existingNodes []*Node
node *Node
action string
expectedIDs map[string]bool
}{
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual.IDs, test.expectedIDs) {
t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs)
}
}
}

481
cluster_test.go
View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"bytes"
"math/rand"
"reflect"
"testing"
@ -29,9 +30,9 @@ import (
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
{URI: test.NewURIFromHostPort("serverA", 1000)},
{URI: test.NewURIFromHostPort("serverB", 1000)},
{URI: test.NewURIFromHostPort("serverC", 1000)},
},
Hasher: test.NewModHasher(),
ReplicaN: 2,
@ -91,52 +92,442 @@ func TestHasher(t *testing.T) {
}
}
// Ensure that an empty cluster returns a valid (empty) NodeSet
func TestCluster_NodeSetHosts(t *testing.T) {
c := pilosa.Cluster{}
if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) {
t.Fatalf("unexpected slice of hosts: %s", h)
}
}
// Ensure cluster can compare its Nodes and Members
func TestCluster_NodeStates(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
NodeSet: &pilosa.StaticNodeSet{},
}
err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{
&pilosa.Node{Host: "serverA:1000"},
&pilosa.Node{Host: "serverC:1000"},
&pilosa.Node{Host: "serverD:1000"},
})
if err != nil {
t.Fatalf("unexpected gossiper nodes: %s", err)
}
// Verify a DOWN node is reported, and extraneous nodes are ignored
if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{
"serverA:1000": pilosa.NodeStateUp,
"serverB:1000": pilosa.NodeStateDown,
"serverC:1000": pilosa.NodeStateUp,
}) {
t.Fatalf("unexpected node state: %s", spew.Sdump(a))
}
}
// Ensure OwnsSlices can find the actual slice list for node and index
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := test.NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := test.NewCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := test.NewURIFromHostPort("node0", 0)
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
uri3 := test.NewURIFromHostPort("node3", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
node3 := &pilosa.Node{ID: "node3", URI: uri3}
nodes := []*pilosa.Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := pilosa.Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs()
expected := []pilosa.URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs()
expected := []pilosa.URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := pilosa.Nodes(nodes).Contains(node1)
actualFalse := pilosa.Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := pilosa.Nodes(nodes).Clone()
actual := pilosa.Nodes(clone).URIs()
expected := []pilosa.URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
c1 := *pilosa.NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *pilosa.NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"}
uri0 := test.NewURIFromHostPort("host0", 0)
uri1 := test.NewURIFromHostPort("host1", 0)
uri2 := test.NewURIFromHostPort("host2", 0)
invalid := test.NewURIFromHostPort("invalid", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := test.NewTestCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != pilosa.ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node2.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{
InverseEnabled: false,
RangeEnabled: true,
//CacheType: pilosa.CacheTypeNone,
Fields: []*pilosa.Field{
{
Name: "fld0",
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
},
},
}); err != nil {
t.Fatal(err)
}
tc.SetFieldValue("i", "fields", 1, "fld0", -10)
tc.SetFieldValue("i", "fields", 1, "fld0", 10)
tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Frame := node0.Holder.Frame("i", "f")
node0View := node0Frame.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
node0Frame = node0.Holder.Frame("i", "fields")
node0View = node0Frame.View("field_fld0")
node0Fragment = node0View.Fragment(1)
node0ChecksumFld := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Values
// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
node1Frame = node1.Holder.Frame("i", "fields")
node1View = node1Frame.View("field_fld0")
node1Fragment = node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := test.NewCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

1
cmd/root_test.go
View file

@ -181,7 +181,6 @@ func TestRootCommand_Config(t *testing.T) {
bind = "127.0.0.1:10101"
[cluster]
poll-interval = "2m0s"
replicas = 2
partitions = 128
hosts = [

12
cmd/server_test.go
View file

@ -45,15 +45,14 @@ func TestServerConfig(t *testing.T) {
// TEST 0
{
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
bind = "localhost:0"
max-writes-per-request = 3000
[cluster]
poll-interval = "45s"
type = "static"
disabled = true
replicas = 2
hosts = [
"localhost:19444",
@ -66,7 +65,6 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.Bind, "localhost:10111")
v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
return v.Error()
@ -80,7 +78,7 @@ func TestServerConfig(t *testing.T) {
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
[cluster]
type = "static"
disabled = true
hosts = [
"localhost:19444",
]
@ -94,13 +92,12 @@ func TestServerConfig(t *testing.T) {
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.type", "static"},
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"},
cfgFileContent: `
bind = "localhost:19444"
data-dir = "` + actualDataDir + `"
[cluster]
poll-interval = "2m0s"
hosts = [
"localhost:19444",
]
@ -116,7 +113,6 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Minute*2))
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
v.Check(cmd.Server.CPUProfile, profFile.Name())
v.Check(cmd.Server.CPUTime, time.Minute)

199
config.go
View file

@ -26,27 +26,92 @@ const (
)
const (
// DefaultDataDir is the default data directory.
DefaultDataDir = "~/.pilosa"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultPort is the default port use with the hostname.
// DefaultPort is the default port to use with the hostname.
DefaultPort = "10101"
// DefaultClusterType sets the node intercommunication method.
DefaultClusterType = ClusterGossip
// DefaultGossipPort indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// DefaultClusterDisabled sets the node intercommunication method.
DefaultClusterDisabled = false
// DefaultMetrics sets the internal metrics to no-op.
DefaultMetrics = "nop"
// DefaultMaxWritesPerRequest is the default number of writes per request.
DefaultMaxWritesPerRequest = 5000
)
// ClusterTypes set of cluster types.
var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterGossip}
// Gossip config based on memberlist.Config.
// Port indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
DefaultGossipStreamTimeout = 10 * time.Second
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
DefaultGossipSuspicionMult = 4
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
DefaultGossipPushPullInterval = 30 * time.Second
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
DefaultGossipProbeInterval = 1 * time.Second
DefaultGossipProbeTimeout = 500 * time.Millisecond
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
DefaultGossipInterval = 200 * time.Millisecond
DefaultGossipNodes = 3
DefaultGossipToTheDeadTime = 30 * time.Second
DefaultMetricPollInterval = 0 * time.Minute
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
@ -62,28 +127,6 @@ type TLSConfig struct {
type Config struct {
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
// GossipPort DEPRECATED
GossipPort string `toml:"gossip-port"`
// GossipSeed DEPRECATED
GossipSeed string `toml:"gossip-seed"`
Gossip struct {
Port string `toml:"port"`
Seed string `toml:"seed"`
Key string `toml:"key"`
} `toml:"gossip"`
Cluster struct {
ReplicaN int `toml:"replicas"`
Type string `toml:"type"`
Hosts []string `toml:"hosts"`
PollInterval Duration `toml:"poll-interval"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
// Limits the number of mutating commands that can be in a single request to
// the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs.
@ -91,67 +134,93 @@ type Config struct {
LogPath string `toml:"log-path"`
// TLS
TLS TLSConfig
Cluster struct {
Disabled bool `toml:"disabled"`
Coordinator bool `toml:"coordinator"`
ReplicaN int `toml:"replicas"`
Hosts []string `toml:"hosts"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
Gossip struct {
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
StreamTimeout Duration `toml:"stream-timeout"`
SuspicionMult int `toml:"suspicion-mult"`
PushPullInterval Duration `toml:"push-pull-interval"`
ProbeTimeout Duration `toml:"probe-timeout"`
ProbeInterval Duration `toml:"probe-interval"`
Nodes int `toml:"nodes"`
Interval Duration `toml:"interval"`
ToTheDeadTime Duration `toml:"to-the-dead-time"`
} `toml:"gossip"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
Metric struct {
Service string `toml:"service"`
Host string `toml:"host"`
PollInterval Duration `toml:"poll-interval"`
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
TLS TLSConfig
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
Bind: DefaultHost + ":" + DefaultPort,
DataDir: DefaultDataDir,
Bind: ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
// LogPath: "",
TLS: TLSConfig{},
}
// Cluster config.
c.Cluster.Disabled = DefaultClusterDisabled
// c.Cluster.Coordinator = false
c.Cluster.ReplicaN = DefaultReplicaN
c.Cluster.Type = DefaultClusterType
c.Cluster.PollInterval = Duration(DefaultPollingInterval)
c.Cluster.Hosts = []string{}
c.Cluster.LongQueryTime = Duration(time.Minute)
// Gossip config.
// c.Gossip.Port = ""
// c.Gossip.Seeds = []string{}
// c.Gossip.Key = ""
c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout)
c.Gossip.SuspicionMult = DefaultGossipSuspicionMult
c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval)
c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout)
c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval)
c.Gossip.Nodes = DefaultGossipNodes
c.Gossip.Interval = Duration(DefaultGossipInterval)
c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime)
// AntiEntropy config.
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
// Metric config.
c.Metric.Service = DefaultMetrics
// c.Metric.Host = ""
c.Metric.PollInterval = Duration(DefaultMetricPollInterval)
c.Metric.Diagnostics = true
c.TLS = TLSConfig{}
return c
}
// Validate that all configuration permutations are compatible with each other.
func (c *Config) Validate() error {
if !StringInSlice(c.Cluster.Type, ClusterTypes) {
return ErrConfigClusterTypeInvalid
if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 {
return ErrConfigClusterEnabledHosts
}
if c.Cluster.Type == ClusterGossip {
if len(c.Cluster.Hosts) > 0 {
bindWithDefaults, err := AddressWithDefaults(c.Bind)
if err != nil {
return err
}
if !c.foundHost(bindWithDefaults) {
return ErrConfigHostsMissing
}
}
}
return nil
}
func (c *Config) foundHost(host *URI) bool {
for _, clusterHost := range c.Cluster.Hosts {
uri, err := NewURIFromAddress(clusterHost)
if err != nil {
continue
}
if host.Equals(uri) {
return true
}
}
return false
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration

24
config_test.go
View file

@ -25,27 +25,15 @@ import (
func Test_NewConfig(t *testing.T) {
c := pilosa.NewConfig()
if c.Cluster.Disabled != pilosa.DefaultClusterDisabled {
t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled)
}
// Ensure that hosts can't be specificed on a non-disabled cluster.
c.Cluster.Hosts = []string{c.Bind, "localhost:10102"}
// Change cluster type from the default (gossip) to an invalid string.
c.Cluster.Type = "invalid-type"
if err := c.Validate(); err != pilosa.ErrConfigClusterTypeInvalid {
t.Fatal(err)
}
// Change cluster type back to gossip.
c.Cluster.Type = pilosa.ClusterGossip
// Check for bind address in cluster hosts.
c.Bind = "localhost:1"
if err := c.Validate(); err != pilosa.ErrConfigHostsMissing {
t.Fatal(err)
}
c.Bind = "localhost:10101"
c.Cluster.ReplicaN = 2
c.GossipSeed = "localhost:14000"
if err := c.Validate(); err != nil {
if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts {
t.Fatal(err)
}
}

6
ctl/backup_test.go
View file

@ -50,9 +50,11 @@ func TestBackupCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")

7
ctl/config_test.go
View file

@ -17,11 +17,12 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
"testing"
"github.com/pilosa/pilosa"
)
func TestConfigCommand_Run(t *testing.T) {
@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) {
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
t.Fatalf("Unexpected config: %s", buf.String())
} else if !strings.Contains(buf.String(), ":10101") {
t.Fatalf("Unexpected config: \n%s", buf.String())
}
}

6
ctl/export_test.go
View file

@ -63,9 +63,11 @@ func TestExportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()

1
ctl/generate_config.go
View file

@ -43,7 +43,6 @@ bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
poll-interval = "2m0s"
replicas = 1
hosts = [
"localhost:10101",

14
ctl/import_test.go
View file

@ -69,9 +69,11 @@ func TestImportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()
@ -85,7 +87,7 @@ func TestImportCommand_Run(t *testing.T) {
}
}
// Ensure that the ImportValue path runs (note: we have specifed a value
// Ensure that the ImportValue path runs (note: we have specified a value
// for cm.Field. Because the handler doesn't return errors (it sends them
// to the logger), we don't get an error returned at `cm.Run()` even though
// we haven't setup frame `f` to be RangeEnabled.
@ -109,9 +111,11 @@ func TestImportCommand_RunValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()

6
ctl/restore_test.go
View file

@ -52,9 +52,11 @@ func TestRestoreCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)

52
ctl/server.go
View file

@ -24,26 +24,44 @@ import (
// BuildServerFlags attaches a set of flags to the command for a server instance.
func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
// TLS
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
// Cluster
flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this?
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path")
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
// AntiEntropy
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
// Metric
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
// CPU Profiling
flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]")
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
}

3
ctl/server_test.go
View file

@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) {
stdin, stdout, stderr := GetIO(buf)
Server := server.NewCommand(stdin, stdout, stderr)
BuildServerFlags(cm, Server)
if cm.Flags().Lookup("gossip-port").Name == "" {
t.Fatal("gossip-port flag is required")
}
if cm.Flags().Lookup("data-dir").Name == "" {
t.Fatal("data-dir flag is required")
}

324
diagnostics.go Normal file
View file

@ -0,0 +1,324 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
)
// Default version check URL.
const (
defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// DiagnosticsCollector represents a collector/sender of diagnostics data.
type DiagnosticsCollector struct {
mu sync.Mutex
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
logOutput io.Writer
server *Server
}
// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
return &DiagnosticsCollector{
host: host,
VersionURL: defaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: &http.Client{Timeout: 10 * time.Second},
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *DiagnosticsCollector) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// Flush sends the current metrics.
func (d *DiagnosticsCollector) Flush() error {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, err := d.encode()
if err != nil {
return err
}
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return err
}
// Intentionally ignoring response body, as user does not need to be notified of error.
defer resp.Body.Close()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *DiagnosticsCollector) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// If version has not changed since the last check, return
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.compareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
}
return nil
}
// compareVersion check version strings.
func (d *DiagnosticsCollector) compareVersion(value string) error {
currentVersion := versionSegments(value)
localVersion := versionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *DiagnosticsCollector) encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *DiagnosticsCollector) Set(name string, value interface{}) {
switch v := value.(type) {
case string:
if v == "" {
// Do not set empty string
return
}
}
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *DiagnosticsCollector) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *DiagnosticsCollector) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// logErr logs the error and returns true if an error exists
func (d *DiagnosticsCollector) logErr(err error) bool {
if err != nil {
d.logOutput.Write([]byte(err.Error()))
return true
}
return false
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.SystemInfo.Uptime()
if !d.logErr(err) {
d.Set("HostUptime", uptime)
}
platform, err := d.server.SystemInfo.Platform()
if !d.logErr(err) {
d.Set("OSPlatform", platform)
}
family, err := d.server.SystemInfo.Family()
if !d.logErr(err) {
d.Set("OSFamily", family)
}
version, err := d.server.SystemInfo.OSVersion()
if !d.logErr(err) {
d.Set("OSVersion", version)
}
kernelVersion, err := d.server.SystemInfo.KernelVersion()
if !d.logErr(err) {
d.Set("OSKernelVersion", kernelVersion)
}
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
memFree, err := d.server.SystemInfo.MemFree()
if !d.logErr(err) {
d.Set("MemFree", memFree)
}
memTotal, err := d.server.SystemInfo.MemTotal()
if !d.logErr(err) {
d.Set("MemTotal", memTotal)
}
memUsed, err := d.server.SystemInfo.MemUsed()
if !d.logErr(err) {
d.Set("MemUsed", memUsed)
}
}
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range d.server.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}
// versionSegments returns the numeric segments of the version as a slice of ints.
func versionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}
// SystemInfo collects information about the host OS.
type SystemInfo interface {
Uptime() (uint64, error)
Platform() (string, error)
Family() (string, error)
OSVersion() (string, error)
KernelVersion() (string, error)
MemFree() (uint64, error)
MemTotal() (uint64, error)
MemUsed() (uint64, error)
}
// NewNopSystemInfo creates a no-op implementation of SystemInfo.
func NewNopSystemInfo() *NopSystemInfo {
return &NopSystemInfo{}
}
// NopSystemInfo is a no-op implementation of SystemInfo.
type NopSystemInfo struct {
}
// Uptime is a no-op implementation of SystemInfo.Uptime.
func (n *NopSystemInfo) Uptime() (uint64, error) {
return 0, nil
}
// Platform is a no-op implementation of SystemInfo.Platform.
func (n *NopSystemInfo) Platform() (string, error) {
return "", nil
}
// Family is a no-op implementation of SystemInfo.Family.
func (n *NopSystemInfo) Family() (string, error) {
return "", nil
}
// OSVersion is a no-op implementation of SystemInfo.OSVersion.
func (n *NopSystemInfo) OSVersion() (string, error) {
return "", nil
}
// KernelVersion is a no-op implementation of SystemInfo.KernelVersion.
func (n *NopSystemInfo) KernelVersion() (string, error) {
return "", nil
}
// MemFree is a no-op implementation of SystemInfo.MemFree.
func (n *NopSystemInfo) MemFree() (uint64, error) {
return 0, nil
}
// MemTotal is a no-op implementation of SystemInfo.MemTotal.
func (n *NopSystemInfo) MemTotal() (uint64, error) {
return 0, nil
}
// MemUsed is a no-op implementation of SystemInfo.MemUsed.
func (n *NopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}

267
diagnostics/diagnostics.go
View file

@ -1,267 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"github.com/sony/gobreaker"
)
// TODO: unique Cluster ID
// Default version check URL.
const (
DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// Diagnostics represents a client to the Pilosa cluster.
type Diagnostics struct {
mu sync.Mutex
wg sync.WaitGroup
closing chan struct{}
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
interval time.Duration
cb *gobreaker.CircuitBreaker
logOutput io.Writer
}
// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port".
func New(host string) *Diagnostics {
return &Diagnostics{
closing: make(chan struct{}),
host: host,
VersionURL: DefaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: http.DefaultClient,
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *Diagnostics) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// SetInterval of the diagnostic go routine and match with the circuit breaker timeout.
func (d *Diagnostics) SetInterval(i time.Duration) {
d.interval = i
}
// schedule start the diagnostics service ticker.
func (d *Diagnostics) schedule() {
ticker := time.NewTicker(d.interval)
defer ticker.Stop()
for {
select {
case <-d.closing:
return
case <-ticker.C:
d.CheckVersion()
d.Flush()
}
}
}
// Flush sends the current metrics.
func (d *Diagnostics) Flush() error {
d.mu.Lock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, _ := d.Encode()
d.mu.Unlock()
_, err := d.cb.Execute(func() (interface{}, error) {
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// TODO verify response
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
})
return err
}
// Open configures the circuit breaker used by the HTTP client.
func (d *Diagnostics) Open() {
var st gobreaker.Settings
if d.interval > 0 {
st.Timeout = d.interval * 2
}
d.cb = gobreaker.NewCircuitBreaker(st)
d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics")
}
// Close notify goroutine to stop.
func (d *Diagnostics) Close() error {
close(d.closing)
d.wg.Wait()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *Diagnostics) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// Same a version as last test
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.CompareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
}
return nil
}
// CompareVersion check version strings.
func (d *Diagnostics) CompareVersion(value string) error {
currentVersion := VersionSegments(value)
localVersion := VersionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *Diagnostics) Encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *Diagnostics) Set(name string, value interface{}) {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *Diagnostics) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *Diagnostics) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *Diagnostics) EnrichWithOSInfo() {
osInfo, err := host.Info()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("HostUptime", osInfo.Uptime)
platform, family, version, err := host.PlatformInformation()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSPlatform", platform)
d.Set("OSFamily", family)
d.Set("OSVersion", version)
kernelVersion, err := host.KernelVersion()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSKernelVersion", kernelVersion)
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *Diagnostics) EnrichWithMemoryInfo() {
memory, err := mem.VirtualMemory()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("MemFree", memory.Free)
d.Set("MemTotal", memory.Total)
d.Set("MemUsed", memory.Used)
}
// VersionSegments returns the numeric segments of the version as a slice of ints.
func VersionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}

43
diagnostics/diagnostics_test.go → diagnostics_internal_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics_test
package pilosa
import (
"encoding/json"
@ -23,25 +23,20 @@ import (
"runtime"
"strings"
"testing"
"github.com/pilosa/pilosa/diagnostics"
)
func TestDiagnosticsClient(t *testing.T) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
d.Open()
defer d.Close()
d.Set("gg", 10)
d.Set("ss", "ss")
data, err := d.Encode()
data, err := d.encode()
if err != nil {
t.Fatal(err)
}
@ -58,7 +53,7 @@ func TestDiagnosticsClient(t *testing.T) {
// Test the metrics after a flush.
d.Flush()
data, err = d.Encode()
data, err = d.encode()
if err != nil {
t.Fatal(err)
}
@ -74,7 +69,7 @@ func TestDiagnosticsClient(t *testing.T) {
func TestDiagnosticsVersion_Parse(t *testing.T) {
version := "0.1.1"
vs := diagnostics.VersionSegments(version)
vs := versionSegments(version)
output := []int{0, 1, 1}
if !reflect.DeepEqual(vs, output) {
@ -83,35 +78,33 @@ func TestDiagnosticsVersion_Parse(t *testing.T) {
}
func TestDiagnosticsVersion_Compare(t *testing.T) {
d := diagnostics.New("localhost:10101")
d.Open()
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "v0.1.1"
d.SetVersion(version)
err := d.CompareVersion("v1.7.0")
err := d.compareVersion("v1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("1.7.0")
err = d.compareVersion("1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("0.7.0")
err = d.compareVersion("0.7.0")
if !strings.Contains(err.Error(), "The latest Minor release is") {
t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.2")
err = d.compareVersion("0.1.2")
if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") {
t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.1")
err = d.compareVersion("0.1.1")
if err != nil {
t.Fatalf("Versions should match")
}
d.SetVersion("v1.7.0")
err = d.CompareVersion("0.7.2")
err = d.compareVersion("0.7.2")
if err != nil {
t.Fatalf("Local version is greater")
}
@ -125,11 +118,9 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
Version: "1.1.1",
})
}))
defer server.Close()
// Create a new client.
d := diagnostics.New("localhost:10101")
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "0.1.1"
d.SetVersion(version)
@ -138,10 +129,6 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
d.CheckVersion()
}
type versionResponse struct {
Version string `json:"version"`
}
func compareJSON(a, b []byte) (bool, error) {
var j1, j2 interface{}
if err := json.Unmarshal(a, &j1); err != nil {
@ -156,12 +143,10 @@ func compareJSON(a, b []byte) (bool, error) {
func BenchmarkDiagnostics(b *testing.B) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
defer d.Close()
prev := runtime.GOMAXPROCS(4)
defer runtime.GOMAXPROCS(prev)

5
docs/README.md Normal file
View file

@ -0,0 +1,5 @@
Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/).
Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search.
Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request!

92
docs/administration.md
View file

@ -5,6 +5,7 @@ nav = [
"Installing in production",
"Importing and Exporting Data",
"Versioning",
"Resizing the Cluster",
"Backup/restore",
]
+++
@ -64,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
```
<div class="note">
<p>Note that you must first create a frame with Range Encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
<p>Note that you must first create a frame with range-encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
</div>
#### Exporting
@ -93,6 +94,71 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ
When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes.
### Resizing the Cluster
If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process.
#### Adding a Node
You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries.
If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately.
#### Removing a Node
In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself.
To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing an `/id` request to the node:
``` request
curl localhost:10102/id
```
``` response
40a891fa-243b-4d71-ae24-4f5c78a0f4b1
```
If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node:
``` request
curl localhost:10101/status
```
``` response
{
"state":"NORMAL",
"nodes":[
{"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
{"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
{"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
]
}
```
Once you have the ID of the node that you want to remove from the cluster, issue the following request:
```
curl localhost:10101/cluster/resize/remove-node \
-X POST \
-d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}'
```
At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries.
Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator).
#### Aborting a Resize Job
If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node.
For example, if your coordinator node is `localhost:10101`, then you can run:
```
curl localhost:10101/cluster/resize/abort -X POST
```
This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began.
#### Changing the Coordinator
In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator.
```
curl localhost:10101/cluster/resize/set-coordinator \
-X POST \
-d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}'
```
### Backup/restore
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster.
@ -112,19 +178,19 @@ Note: This will only work when the replication factor is >= 2
#### Copying data files manually
- To accomplish this goal you will 1st need:
- List of all Indexes on your cluster
- List of all frames in your Indexes
- Max slice per Index, listed in the /status endpoint
- To accomplish this you will first need:
- List of all indexes on your cluster
- List of all frames in your indexes
- Max slice per index, listed in the /status endpoint
- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
- Using the list of slices owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Frame
- copy each owned slice for an existing node to this new node
- Modify the cluster config file to replace the previous node address with the new node address.
- Restart the cluster
- Wait for the 1st sync (10 minutes) to validate Index connections
- Wait for the first sync (10 minutes) to validate Index connections
#### Diagnostics
### Diagnostics
Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
@ -145,16 +211,16 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
You can opt-out of the Pilosa diagnostics reporting by setting either the command line configuration option `--metric.diagnostics=false`, use the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
#### Metrics
### Metrics
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
The metrics configuration options are:
- [Host](../configuration#metrics-host): specify host that receives metric events
- [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics
- [Service](../configuration#metrics-service): declare type StatsD or Expvar
- [Host](../configuration/#metric-host): specify host that receives metric events
- [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
- [Service](../configuration/#metric-service): declare type StatsD or Expvar
##### Tags
#### Tags
StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- NodeID
@ -163,7 +229,7 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- View
- Slice
##### Events
#### Events
We currently track the following events
- **Index:** The creation of a new Index.

33
docs/api-reference.md
View file

@ -78,7 +78,7 @@ In order to send protobuf binaries in the request and response, set `Content-Typ
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
@ -102,14 +102,15 @@ Creates a frame in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `rangeEnabled` (boolean): Enables range-encoded fields in this frame.
* `fields` (array): List of range-encoded fields.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:
* `name` (string): Field name.
* `type` (string): Field type, currently only "int" is supported.
* `min` (int): Minimum value allowed for this field.
@ -199,15 +200,19 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \
### Create input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`POST /index/<index-name>/input-definition/<input-definition-name>`
Creates an input definition in the given index with the given name.
The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options:
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa.
@ -263,6 +268,10 @@ curl localhost:10101/index/user/input-definition/stargazer-input \
### Get input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`GET /index/<index-name>/input-definition/<input-definition-name>`
Returns the given input definition as JSON.
@ -276,6 +285,10 @@ curl -XGET localhost:10101/index/user/input-definition/stargazer-input
### Remove input definition
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`DELETE /index/<index-name>/input-definition/<input-definition-name>`
Removes the given input definition.
@ -289,6 +302,10 @@ curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
### Process input data
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
`POST /index/<index-name>/input/<input-definition-name>`
Processes the JSON payload using the given input definition.

3
docs/architecture.md
View file

@ -18,5 +18,8 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B
* After the container storage section is an operation log, of unspecified length.
![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png)
*Pilosa Roaring storage format diagram*
All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.
Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa.

9
docs/client-libraries.md
View file

@ -10,12 +10,13 @@ nav = [
## Client Libraries
This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is not wise to use the output of a hash, or randomly distributed ids with Pilosa.
### Go
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
@ -95,9 +96,9 @@ func main() {
### Python
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions.
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
@ -170,7 +171,7 @@ client.query(stargazer.setbit(99999, 77777))
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.

91
docs/configuration.md
View file

@ -27,24 +27,18 @@ Every command line flag has a corresponding environment variable. The environmen
### Config file
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file:
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file:
```toml
[cluster]
poll-interval = "2m0s"
coordinator = true
replicas = 1
```
Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes:
```toml
[cluster]
hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"]
```
### All Options
#### Anti Entropy Interval
* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync.
* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
* Flag: `--anti-entropy.interval="10m0s"`
* Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"`
* Config:
@ -84,7 +78,7 @@ Any flag that has a value that is a comma separated list on the command line bec
* Config:
```toml
log_path = "/path/to/logfile"
log-path = "/path/to/logfile"
```
#### Max Writes Per Request
@ -110,16 +104,16 @@ Any flag that has a value that is a comma separated list on the command line bec
port = 11101
```
#### Gossip Seed
#### Gossip Seeds
* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`.
* Flag: `--gossip.seed="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEED="localhost:11101"`
* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms.
* Flag: `--gossip.seeds="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101"`
* Config:
```toml
[gossip]
seed = "localhost:11101"
seeds = ["localhost:11101"]
```
#### Gossip Key
@ -133,28 +127,16 @@ Any flag that has a value that is a comma separated list on the command line bec
key = "/var/secret/gossip.key32"
```
#### Cluster Hosts
#### Cluster Coordinator
* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms.
* Flag: `--cluster.hosts="localhost:10101"`
* Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"`
* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
* Flag: `cluster.coordinator`
* Env: `PILOSA_CLUSTER_COORDINATOR`
* Config:
```toml
[cluster]
hosts = ["localhost:10101"]
```
#### Cluster Poll Interval
* Description: Polling interval for cluster.
* Flag: `cluster.poll-interval="1m0s"`
* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"`
* Config:
```toml
[cluster]
poll-interval = "1m0s"
coordinator = true
```
#### Cluster Long Query Time
@ -183,9 +165,8 @@ Any flag that has a value that is a comma separated list on the command line bec
#### Cluster Type
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip].
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip].
* static - Messaging between nodes is disabled. This is primarily used for testing.
* http - Messages are transmitted over HTTP.
* gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip.
* Flag: `cluster.type="gossip"`
* Env: `PILOSA_CLUSTER_TYPE="gossip"`
@ -219,7 +200,8 @@ Any flag that has a value that is a comma separated list on the command line bec
[profile]
cpu-time = "30s"
```
##### Metric Service
#### Metric Service
* Description: Which stats service to use. Choose from [statsd, expvar].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd'
@ -230,7 +212,7 @@ Any flag that has a value that is a comma separated list on the command line bec
service = “statsd”
```
##### Metric Host
#### Metric Host
* Description: Address of the StatsD service host.
* Flag: `--metric.host=localhost:8125`
* Env: `PILOSA_METRIC_HOST=localhost:8125'
@ -241,7 +223,7 @@ Any flag that has a value that is a comma separated list on the command line bec
host = "localhost:8125"
```
##### Metric Poll Interval
#### Metric Poll Interval
* Description: Polling interval for runtime metrics.
* Flag: `metric.poll-interval=”0m15s”`
@ -253,7 +235,7 @@ Any flag that has a value that is a comma separated list on the command line bec
poll-interval = "0m15s"
```
##### Metric Diagnostics
#### Metric Diagnostics
* Description: Enable diagnostic reporting. To disable diagnostics set to false.
* Flag: `metric.diagnostics`
@ -266,7 +248,7 @@ Any flag that has a value that is a comma separated list on the command line bec
```
##### TLS Certificate
#### TLS Certificate
* Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of`.crt` or `.pem` extensions.
* Flag: `tls.certificate=/srv/pilosa/certs/server.crt`
@ -278,7 +260,7 @@ Any flag that has a value that is a comma separated list on the command line bec
certificate = "/srv/pilosa/certs/server.crt"
```
##### TLS Certificate Key
#### TLS Certificate Key
* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension.
* Flag: `tls.key=/srv/pilosa/certs/server.key`
@ -290,7 +272,7 @@ Any flag that has a value that is a comma separated list on the command line bec
key = "/srv/pilosa/certs/server.key"
```
##### TLS Skip Verify
#### TLS Skip Verify
* Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial.
* Flag: `tls.skip-verify`
@ -317,8 +299,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = true
#### Node 1
@ -331,8 +312,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
#### Node 2
@ -345,8 +325,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
### Example Cluster Configuration (HTTPS)
@ -365,8 +344,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -384,8 +362,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -403,8 +380,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -426,8 +402,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -445,8 +420,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -464,8 +438,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"

30
docs/data-model.md
View file

@ -24,7 +24,10 @@ Rows and columns can represent anything (they could even represent the same set
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row.
![data model diagram](/img/docs/data-model.svg)
Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen.
![basic data model diagram](/img/docs/data-model.svg)
*Basic data model diagram*
### Index
@ -49,16 +52,18 @@ Row attributes are namespaced at the Frame level.
Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
![ranked frame diagram](/img/docs/frame-ranked.svg)
*Ranked frame diagram*
#### LRU
The LRU cache maintains the most recently accessed Rows.
![lru frame diagram](/img/docs/frame-lru.svg)
*LRU frame diagram*
### Time Quantum
Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
### Attribute
@ -90,6 +95,7 @@ SetBit(frame="A", rowID=19, columnID=5)
```
![inverse frame diagram](/img/docs/frame-inverse.svg)
*Inverse frame diagram*
#### Time Quantums
@ -101,23 +107,27 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
```
![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
*Time quantum frame diagram*
#### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate, aggregate, and range queries on these BSI integers.
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
For example, the following `SetFieldValue()` queries will result in the data described in the illustration below:
```
SetFieldValue(col=1, frame="A", field0=1)
SetFieldValue(col=2, frame="A", field0=2)
SetFieldValue(col=3, frame="A", field0=3)
SetFieldValue(col=4, frame="A", field0=7)
SetFieldValue(col=2, frame="A", field1=1)
SetFieldValue(col=3, frame="A", field1=6)
SetFieldValue(columnID=1, frame="A", field0=1)
SetFieldValue(columnID=2, frame="A", field0=2)
SetFieldValue(columnID=3, frame="A", field0=3)
SetFieldValue(columnID=4, frame="A", field0=7)
SetFieldValue(columnID=2, frame="A", field1=1)
SetFieldValue(columnID=3, frame="A", field1=6)
```
![BSI diagram](/img/docs/frame-bsi.svg)
![BSI frame diagram](/img/docs/frame-bsi.svg)
*BSI frame diagram*
Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.

8
docs/examples.md
View file

@ -17,7 +17,7 @@ New York City released an extremely detailed data set of over 1 billion taxi rid
Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
@ -163,7 +163,7 @@ durm := pdk.CustomMapper{
#### Import process
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk/) for more details on this process.
#### Queries
@ -214,7 +214,7 @@ T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase).
#### Data model
@ -263,7 +263,7 @@ python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
```
First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
```
curl localhost:10101/index/mole \

26
docs/getting-started.md
View file

@ -20,7 +20,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th
### Starting Pilosa
Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa.
Follow the steps in the [Install](../installation/) document to install Pilosa.
Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`):
```
pilosa server
@ -35,18 +35,20 @@ Let's make sure Pilosa is running:
curl localhost:10101/status
```
``` response
{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}
{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]}
```
### Sample Project
In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation.
#### Create the Schema
The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
Note:
The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
``` request
curl localhost:10101/schema
```
@ -85,7 +87,7 @@ curl localhost:10101/index/repository/frame/language \
#### Import Data From CSV Files
If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo.
Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
@ -107,10 +109,15 @@ docker cp language.csv pilosa:/language.csv
docker exec -it pilosa /pilosa import -i repository -f language /language.csv
```
Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages.
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
### Input Definition
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
<div class="warning">
Input definition is deprecated as of v0.9.
</div>
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
#### Make Some Queries
@ -226,6 +233,11 @@ curl localhost:10101/index/repository/query \
{"results":[true]}
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to poor performance, out of memory errors, and more.
### What's Next?
You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).

61
docs/glossary.md
View file

@ -6,53 +6,62 @@ nav = []
## Glossary
<strong id="index">Index:</strong> Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
<strong id="column">Column:</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index.
<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
<strong id="row">Row:</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index.
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column).
<strong id="bit">Bit:</strong> A bit is the intersection of a Row and Column.
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
<strong id="bitmap">Bitmap:</strong> The on-disk and in-memory representation of a Row.
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="roaring-bitmap">Roaring Bitmap:</strong> [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="attribute">Attribute:</strong> Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store.
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index).
<strong id="pql">PQL:</strong> Pilosa Query Language
<strong id="field">[Field](../data-model/#bsi-range-encoding):</strong> A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="index">Index:</strong> The Index represents a data namespace.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
<strong id="frame">Frame:</strong> Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame.
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
<strong id="view">View:</strong> Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a frame and slice in an index.
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="slice">Slice:</strong> Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash.
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
<strong id="slicewidth">SliceWidth:</strong> This is the default number of columns in a slice.
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
<strong id="maxslice">MaxSlice:</strong> The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries.
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
<strong id="anti-entropy">Anti-entropy:</strong> A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies.
<strong id="pql">[PQL](../query-language/):</strong> Pilosa Query Language.
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a cluster.
<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="range">[Range](../query-language/#range-queries):</strong>: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
<strong id="topn">TopN:</strong> Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row.
<strong id="range-bsi">[Range (BSI)](../query-language/#range-bsi):</strong>: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
<strong id="tanimoto">Tanimoto:</strong> Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity.
<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
<strong id="protobuf">Protobuf:</strong>: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
<strong id="toml">TOML:</strong> We use [TOML](https://github.com/toml-lang/toml) for our configuration file format.
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="jump-consistent-hash">Jump Consistent Hash:</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
https://arxiv.org/pdf/1406.2294v1.pdf
<strong id="slice">[Slice](../data-model/#slice):</strong> [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
<strong id="partition">Partition:</strong> The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography.
<strong id="slicewidth">SliceWidth:</strong> This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
<strong id="replica">Replica:</strong> A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made).
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field).
<strong id="tanimoto">[Tanimoto](../examples/#chemical-similarity-search):</strong> Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for time [Range](#range) queries.
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

63
docs/input-definition.md
View file

@ -1,64 +1,11 @@
+++
title = "Input Definition"
weight = 8
nav = [
"Create the Schema",
"Import Data",
]
+++
## Input Definition
This document builds on the data import concepts introduced in [Getting Started](../getting-started/).
Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data.
### Create the Schema
Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations.
Before creating a schema, let's create the repository index first:
```
curl localhost:10101/index/repository -X POST
```
The sample input definition schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started) in the `input_definition.json` file. Download it using:
```
curl -OL https://github.com/pilosa/getting-started/raw/master/input_definition.json
```
Run the following to create the input definition:
```
curl localhost:10101/index/repository/input-definition/stargazer -d @input_definition.json
```
Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition.
We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination:
- value-to-row: The value for this field is used as the `rowID`.
- single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type.
- mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type.
- set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame
### Import Data
The sample data for the input definition we created above is in the `json_input.json` file at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download it using:
```
curl -OL https://github.com/pilosa/getting-started/raw/master/json_input.json
```
Then run the following to import it:
```
curl localhost:10101/index/repository/input/stargazer -d @json_input.json
```
As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID.
The data input above is equivalent to the following `SetBit()` operations:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=91720568, rowID=513114)
SetBit(frame="stargazer", columnID=91720568, rowID=513114, timestamp="2017-05-18T20:40")
SetBit(frame="language", columnID=91720568, rowID=5)
SetBit(frame="language", columnID=95122322, rowID=17)
'
```
<div class="warning">
Input definition is deprecated as of Pilosa v0.9.<br/>
<br/>
The previous version of this page is still available <a href="https://www.pilosa.com/docs/v0.8/input-definition">here</a>.
</div>

17
docs/installation.md
View file

@ -136,15 +136,16 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa):
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make generate-statik
@ -203,7 +204,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
docker version
```
If you don't see the server listed, start the Docker application.
If you don't see the server listed, start the Docker application.
3. Pull the official Pilosa image from Docker Hub:
```
@ -289,17 +290,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo:
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
make generate-statik
make install
```

2
docs/introduction.md
View file

@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w
"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.
Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.

99
docs/query-language.md
View file

@ -13,7 +13,7 @@ nav = [
### Overview
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
```
{"results":[...]}
@ -31,7 +31,7 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch
##### Examples
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data.
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
``` request
@ -45,14 +45,14 @@ curl localhost:10101/index/repository/query \
#### Arguments and Types
* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated.
* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated.
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
* `UINT` An unsigned integer (e.g. 42839)
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
* `ATTR_VALUE` Can be a string, float, integer, or bool.
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range`
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
### Write Operations
@ -80,14 +80,14 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
**Examples:**
```
SetBit(frame="stargazer", repo_id=10, rowID=1)
SetBit(frame="stargazer", columnID=10, rowID=1)
```
This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10.
SetBit also supports providing a timestamp. To write the date that a user starred a repository.
```
SetBit(frame="stargazer", repo_id=10, rowID=1, timestamp="2016-01-01T00:00")
SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00")
```
Setting multiple bits in a single request:
@ -118,7 +118,7 @@ SetRowAttrs queries always return `null` upon success.
SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true)
```
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
```
SetRowAttrs(frame="stargazer", rowID=10, username=null)
@ -150,7 +150,7 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu
SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true)
```
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`.
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`.
```
SetColumnAttrs(columnID=10, url=null)
@ -184,7 +184,31 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
ClearBit(frame="stargazer", columnID=10, rowID=1)
```
Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame.
Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame.
#### SetFieldValue
**Spec:**
```
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
**Result Type:** null
SetFieldValue returns `null` upon success.
**Examples:**
Set the number of pull requests of repository 10.
```
SetFieldValue(columnID=10, frame="stats", pullrequests=2)
```
### Read Operations
@ -308,6 +332,34 @@ Return `{"attrs":{},"bits":[30]}`
* Bits are repositories that were starred by user 2 BUT NOT user 1
#### Xor
**Spec:**
```
Xor(<BITMAP_CALL>, [BITMAP_CALL ...])
```
**Description:**
Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it.
**Result Type:** object with attrs and bits
attrs will always be empty
**Examples:**
Query repositories which have been starred by two users.
```
Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2))
```
Returns `{"attrs":{},"bits":[30]}`.
* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
#### Count
**Spec:**
@ -352,6 +404,7 @@ have the attribute specified by `field` with one of the values specified in
**Result Type:** array of key/count objects
**Caveats:**
* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order.
* Frames with cache type lru will maintain an LRU (Least Recently Used) cache, thus a TopN() query on this type of frame will return bitmaps sorted in order of most recently set bit.
* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN() queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
@ -444,7 +497,7 @@ Returns bits that are true for the comparison operator.
**Examples:**
In our source data, commitactivity was counted over the last year.
The following greater-than Range query returns all repositories having more than 100 commits.
The following greater-than `Range` query returns all repositories having more than 100 commits.
```
Range(frame="stats", commitactivity > 100)
@ -484,7 +537,7 @@ Sum([BITMAP_CALL], <frame=STRING>, <field=STRING>)
**Description:**
Returns the count and computed sum of all bitmap encoded integer values across the `field` in this `frame`. The optional Bitmap call filters the bits used in this computation.
Returns the count and computed sum of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
**Result Type:** object with the computed sum and count of the bitmap field.
@ -498,27 +551,3 @@ Sum(frame="stats", field="diskusage")
Return `{"sum":10,"count":3}`
* Result is the size of all repositories in kilobytes, plus the number of repositories.
#### SetFieldValue
**Spec:**
```
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
**Result Type:** null
SetFieldValue returns `null` upon success.
**Examples:**
Set the number of pull requests of repository 10.
```
SetFieldValue(col=10, frame="stats", pullrequests=2)
```

18
docs/tutorials.md
View file

@ -10,6 +10,16 @@ nav = [
## Tutorials
<div class="note">
<!-- this is html because there is a problem putting a list inside a shortcode -->
Some of our tutorials work better as standalone repos, since you can <code>git clone</code> the instructions, code, and data all at once. Officially supported tutorials are listed here.<br />
<br />
<ul>
<li><a href="https://github.com/pilosa/cosmosa">Run Pilosa with Microsoft's Azure Cosmos DB</a></li>
</ul>
</div>
### Setting Up a Secure Cluster
#### Introduction
@ -20,7 +30,7 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma
#### Installing Pilosa and Creating the Directory Structure
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source).
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
```
@ -45,6 +55,7 @@ openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.cr
```
The command above creates two files in the current directory:
* `pilosa.local.crt` is the SSL certificate.
* `pilosa.local.key` is the private key file which must be kept as secret.
@ -126,9 +137,10 @@ key = "pilosa.local.gossip32"
```
Here is some explanation of the configuration items:
* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings.
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings.
* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
@ -235,7 +247,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
``` request

11
docs/webui.md
View file

@ -9,18 +9,20 @@ nav = [
## WebUI
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101)
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
This can be used for constructing queries and viewing the cluster status.
### Console
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
The Console will keep a record of each query and its result with the latest query on top.
![console](/img/docs/webui-console.png)
![webUI console screenshot](/img/docs/webui-console.png)
*WebUI console screenshot*
In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
@ -30,9 +32,8 @@ In addition to standard PQL, the console supports a few special commands, prefix
- `:create frame <framename>`
- `:delete frame <framename>`
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference).
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
- `:create index <indexname> timeQuantum=YM`
- `:create frame <framename> inverseEnabled=true cacheSize=10000`

55
event.go Normal file
View file

@ -0,0 +1,55 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// NodeEventType are the types of events that can be sent from the
// ChannelEventDelegate.
type NodeEventType int
const (
NodeJoin NodeEventType = iota
NodeLeave
NodeUpdate
)
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
}
// EventHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type EventHandler interface {
ReceiveEvent(e *NodeEvent) error
}
// EventReceiver is the interface for the object which will listen for and
// decode broadcast messages before passing them to pilosa to handle. The
// implementation of this could be an http server which listens for messages,
// gets the protobuf payload, and then passes it to
// EventHandler.ReceiveMessage.
type EventReceiver interface {
// Start starts listening for broadcast messages - it should return
// immediately, spawning a goroutine if necessary.
Start(EventHandler) error
}
type nopEventReceiver struct{}
func (n *nopEventReceiver) Start(e EventHandler) error { return nil }
// NopEventReceiver is a no-op implementation of the EventReceiver.
var NopEventReceiver = &nopEventReceiver{}

29
executor.go
View file

@ -40,8 +40,7 @@ type Executor struct {
Holder *Holder
// Local hostname & cluster configuration.
Scheme string
Host string
Node *Node
Cluster *Cluster
// Client used for remote requests.
@ -801,7 +800,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return NewBitmap(), nil
}
// LT[E] and GT[E] should return all not-null if selected range fully encompases valid field range.
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range.
if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) ||
(cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) {
return frag.FieldNotNull(field.BitDepth())
@ -957,7 +956,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
if err != nil {
return false, err
@ -1062,7 +1061,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
if err != nil {
return false, err
@ -1141,7 +1140,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1199,7 +1198,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1286,7 +1285,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1345,7 +1344,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1372,12 +1371,8 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
Slices: slices,
Remote: true,
}
uri, err := NewURIFromAddress(node.Host)
if err != nil {
return nil, err
}
uri.SetScheme(node.Scheme)
ctx = context.WithValue(ctx, "uri", uri)
ctx = context.WithValue(ctx, "uri", node.URI)
pb, err := e.client.ExecuteQuery(ctx, index, pbreq)
if err != nil {
return nil, err
@ -1456,7 +1451,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.Nodes).Clone()
} else {
nodes = []*Node{e.Cluster.NodeByHost(e.Host)}
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
@ -1512,7 +1507,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
resp := mapResponse{node: n, slices: nodeSlices}
// Send local slices to mapper, otherwise remote exec.
if n.Host == e.Host {
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)

40
executor_test.go
View file

@ -897,8 +897,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Scheme = "http"
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -941,7 +945,13 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -971,7 +981,13 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
@ -1017,7 +1033,13 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
@ -1064,7 +1086,13 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int

42
filesystem.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"net/http"
)
// Ensure nopFileSystem implements interface.
var _ FileSystem = &nopFileSystem{}
// FileSystem represents an interface for a WebUI file system.
type FileSystem interface {
New() (http.FileSystem, error)
}
func init() {
NopFileSystem = &nopFileSystem{}
}
// NopFileSystem represents a FileSystem that returns an error if called.
var NopFileSystem FileSystem
type nopFileSystem struct{}
// New is a no-op implementation of FileSystem New method.
func (n *nopFileSystem) New() (http.FileSystem, error) {
return nil, fmt.Errorf("file system not implemented")
}

23
fragment.go
View file

@ -108,7 +108,7 @@ type Fragment struct {
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
RowAttrStore *AttrStore
RowAttrStore AttrStore
stats StatsClient
}
@ -256,6 +256,7 @@ func (f *Fragment) openCache() error {
f.cache = NewLRUCache(f.CacheSize)
case CacheTypeNone:
f.cache = NewNopCache()
return nil
default:
return ErrInvalidCacheType
}
@ -1455,6 +1456,10 @@ func (f *Fragment) flushCache() error {
return nil
}
if f.CacheType == CacheTypeNone {
return nil
}
// Retrieve a list of row ids from the cache.
ids := f.cache.IDs()
@ -1681,7 +1686,7 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Host string
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -1711,17 +1716,14 @@ func (s *FragmentSyncer) SyncFragment() error {
blockSets := make([][]FragmentBlock, 0, len(nodes))
for _, node := range nodes {
// Read local blocks.
if node.Host == s.Host {
if node.ID == s.Node.ID {
b := s.Fragment.Blocks()
blockSets = append(blockSets, b)
continue
}
// Retrieve remote blocks.
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return err
@ -1788,7 +1790,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
var pairSets []PairSet
var clients []InternalClient
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
if s.Host == node.Host {
if s.Node.ID == node.ID {
continue
}
@ -1797,10 +1799,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
return nil
}
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
clients = append(clients, client)
// Only sync the standard block.

4
fragment_test.go
View file

@ -163,7 +163,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
t.Fatal("expected change")
}
// Non-existant value.
// Non-existent value.
if value, exists, err := f.FieldValue(100, 11); err != nil {
t.Fatal(err)
} else if value != 0 {
@ -702,7 +702,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
Fragment: frag,
RowAttrStore: test.MustOpenAttrStore(),
}
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
panic(err)
}

310
frame.go
View file

@ -43,29 +43,27 @@ const (
// Frame represents a container for views.
type Frame struct {
mu sync.RWMutex
path string
index string
name string
timeQuantum TimeQuantum
schema *FrameSchema
mu sync.RWMutex
path string
index string
name string
views map[string]*View
// Row attribute storage and cache
rowAttrStore *AttrStore
rowAttrStore AttrStore
broadcaster Broadcaster
Stats StatsClient
// Frame settings.
// Frame options.
rowLabel string
cacheType string
inverseEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
rangeEnabled bool
// Cache size for ranked frames
cacheSize uint32
fields []*Field
LogOutput io.Writer
}
@ -78,22 +76,24 @@ func NewFrame(path, index, name string) (*Frame, error) {
}
return &Frame{
path: path,
index: index,
name: name,
schema: &FrameSchema{},
path: path,
index: index,
name: name,
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
views: make(map[string]*View),
rowAttrStore: NopAttrStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
rowLabel: DefaultRowLabel,
inverseEnabled: DefaultInverseEnabled,
rangeEnabled: DefaultRangeEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
rangeEnabled: DefaultRangeEnabled,
//fields
LogOutput: ioutil.Discard,
}, nil
@ -109,7 +109,7 @@ func (f *Frame) Index() string { return f.index }
func (f *Frame) Path() string { return f.path }
// RowAttrStore returns the attribute storage.
func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
@ -230,7 +230,7 @@ func (f *Frame) options() FrameOptions {
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.schema.Fields,
Fields: f.fields,
}
}
@ -244,8 +244,6 @@ func (f *Frame) Open() error {
if err := f.loadMeta(); err != nil {
return err
} else if err := f.loadSchema(); err != nil {
return err
}
if err := f.openViews(); err != nil {
@ -304,12 +302,13 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.timeQuantum = ""
f.rowLabel = DefaultRowLabel
f.cacheType = DefaultCacheType
f.inverseEnabled = DefaultInverseEnabled
f.rangeEnabled = DefaultRangeEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
f.rangeEnabled = DefaultRangeEnabled
//f.fields
return nil
} else if err != nil {
return err
@ -320,17 +319,16 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rowLabel = pb.RowLabel
f.inverseEnabled = pb.InverseEnabled
f.rangeEnabled = pb.RangeEnabled
f.cacheSize = pb.CacheSize
// Copy cache type.
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
}
f.cacheSize = pb.CacheSize
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rangeEnabled = pb.RangeEnabled
f.fields = decodeFields(pb.Fields)
return nil
}
@ -352,35 +350,6 @@ func (f *Frame) saveMeta() error {
return nil
}
// loadSchema reads the schema for the frame.
func (f *Frame) loadSchema() error {
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema"))
if os.IsNotExist(err) {
f.schema = &FrameSchema{}
return nil
} else if err != nil {
return err
}
var pb internal.FrameSchema
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
f.schema = decodeFrameSchema(&pb)
return nil
}
// saveSchema writes the current schema to disk.
func (f *Frame) saveSchema() error {
if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil {
return err
}
return nil
}
// Close closes the frame and its views.
func (f *Frame) Close() error {
f.mu.Lock()
@ -402,16 +371,11 @@ func (f *Frame) Close() error {
return nil
}
// Schema returns the frame's current schema.
func (f *Frame) Schema() *FrameSchema {
// Field returns a field by name.
func (f *Frame) Field(name string) *Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.schema
}
// Field returns a field from the schema by name.
func (f *Frame) Field(name string) *Field {
for _, field := range f.Schema().Fields {
for _, field := range f.fields {
if field.Name == name {
return field
}
@ -419,7 +383,24 @@ func (f *Frame) Field(name string) *Field {
return nil
}
// CreateField creates a new field on the schema.
// Fields returns the fields on the frame.
func (f *Frame) Fields() []*Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.fields
}
// HasField returns true if a field exists on the frame.
func (f *Frame) HasField(name string) bool {
for _, fld := range f.fields {
if fld.Name == name {
return true
}
}
return false
}
// CreateField creates a new field on the frame.
func (f *Frame) CreateField(field *Field) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -429,18 +410,35 @@ func (f *Frame) CreateField(field *Field) error {
return ErrFrameFieldsNotAllowed
}
// Copy schema and append field.
schema := f.schema.Clone()
if err := schema.AddField(field); err != nil {
// Append field.
if err := f.addField(field); err != nil {
return err
}
f.schema = schema
f.saveSchema()
f.saveMeta()
return nil
}
// addField adds a single field to fields.
func (f *Frame) addField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if f.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
f.fields = append(f.fields, field)
// Sort fields by name.
sort.Slice(f.fields, func(i, j int) bool {
return f.fields[i].Name < f.fields[j].Name
})
return nil
}
// GetFields returns a list of all the fields in the frame.
func (f *Frame) GetFields() (*FrameSchema, error) {
func (f *Frame) GetFields() ([]*Field, error) {
f.mu.RLock()
defer f.mu.RUnlock()
@ -449,12 +447,12 @@ func (f *Frame) GetFields() (*FrameSchema, error) {
return nil, ErrFrameFieldsNotAllowed
}
err := f.loadSchema()
err := f.loadMeta()
if err != nil {
return nil, err
}
return f.schema, nil
return f.fields, nil
}
// DeleteField deletes an existing field on the schema.
@ -467,12 +465,10 @@ func (f *Frame) DeleteField(name string) error {
return ErrFrameFieldsNotAllowed
}
// Copy schema and remove field.
schema := f.schema.Clone()
if err := schema.DeleteField(name); err != nil {
// Remove field.
if err := f.deleteField(name); err != nil {
return err
}
f.schema = schema
// Remove views.
viewName := ViewFieldPrefix + name
@ -489,6 +485,18 @@ func (f *Frame) DeleteField(name string) error {
return nil
}
// deleteField removes a single field from fields.
func (f *Frame) deleteField(name string) error {
for i, field := range f.fields {
if field.Name == name {
copy(f.fields[i:], f.fields[i+1:])
f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
f.mu.Lock()
@ -543,6 +551,18 @@ func (f *Frame) Views() []*View {
return other
}
// viewNames returns a list of all views (as a string) in the frame.
func (f *Frame) viewNames() []string {
f.mu.Lock()
defer f.mu.Unlock()
other := make([]string, 0, len(f.views))
for viewName, _ := range f.views {
other = append(other, viewName)
}
return other
}
// RecalculateCaches recalculates caches on every view in the frame.
func (f *Frame) RecalculateCaches() {
for _, view := range f.Views() {
@ -551,27 +571,54 @@ func (f *Frame) RecalculateCaches() {
}
// CreateViewIfNotExists returns the named view, creating it if necessary.
// Additionally, a CreateViewMessage is sent to the cluster.
func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
view, created, err := f.createViewIfNotExistsBase(name)
if err != nil {
return nil, err
}
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&internal.CreateViewMessage{
Index: f.index,
Frame: f.name,
View: name,
})
if err != nil {
return nil, err
}
}
return view, nil
}
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, ErrFrameInverseDisabled
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
if view := f.views[name]; view != nil {
return view, nil
return view, false, nil
}
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return nil, err
return nil, false, err
}
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
return view, nil
return view, true, nil
}
func (f *Frame) newView(path, name string) *View {
@ -956,8 +1003,9 @@ func encodeFrames(a []*Frame) []*internal.Frame {
func encodeFrame(f *Frame) *internal.Frame {
fo := f.options()
return &internal.Frame{
Name: f.name,
Meta: fo.Encode(),
Name: f.name,
Meta: fo.Encode(),
Views: f.viewNames(),
}
}
@ -969,8 +1017,9 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FrameInfo represents schema information for a frame.
type FrameInfo struct {
Name string `json:"name"`
Views []*ViewInfo `json:"views,omitempty"`
Name string `json:"name"`
Options FrameOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type frameInfoSlice []*FrameInfo
@ -992,6 +1041,13 @@ type FrameOptions struct {
// Encode converts o into its internal representation.
func (o *FrameOptions) Encode() *internal.FrameMeta {
return encodeFrameOptions(o)
}
func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
if o == nil {
return nil
}
return &internal.FrameMeta{
RowLabel: o.RowLabel,
InverseEnabled: o.InverseEnabled,
@ -1003,74 +1059,18 @@ func (o *FrameOptions) Encode() *internal.FrameMeta {
}
}
// FrameSchema represents the list of fields on a frame.
type FrameSchema struct {
Fields []*Field
}
// Clone returns a copy of s.
func (s *FrameSchema) Clone() *FrameSchema {
other := &FrameSchema{Fields: make([]*Field, len(s.Fields))}
copy(other.Fields, s.Fields)
return other
}
// HasField returns true if a field exists on the schema.
func (s *FrameSchema) HasField(name string) bool {
for _, f := range s.Fields {
if f.Name == name {
return true
}
}
return false
}
// AddField adds a single field to the schema.
func (s *FrameSchema) AddField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if s.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
s.Fields = append(s.Fields, field)
// Sort fields by name.
sort.Slice(s.Fields, func(i, j int) bool {
return s.Fields[i].Name < s.Fields[j].Name
})
return nil
}
// DeleteField removes a single field from the schema.
func (s *FrameSchema) DeleteField(name string) error {
for i, field := range s.Fields {
if field.Name == name {
copy(s.Fields[i:], s.Fields[i+1:])
s.Fields, s.Fields[len(s.Fields)-1] = s.Fields[:len(s.Fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema {
if schema == nil {
func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
if options == nil {
return nil
}
return &internal.FrameSchema{
Fields: encodeFields(schema.Fields),
}
}
func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema {
if schema == nil {
return nil
}
return &FrameSchema{
Fields: decodeFields(schema.Fields),
return &FrameOptions{
RowLabel: options.RowLabel,
InverseEnabled: options.InverseEnabled,
RangeEnabled: options.RangeEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

41
gc.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// Ensure nopGCNotifier implements interface.
var _ GCNotifier = &nopGCNotifier{}
// GCNotifier represents an interface for garbage collection notificationss.
type GCNotifier interface {
Close()
AfterGC() <-chan struct{}
}
func init() {
NopGCNotifier = &nopGCNotifier{}
}
// NopGCNotifier represents a GCNotifier that doesn't do anything.
var NopGCNotifier GCNotifier
type nopGCNotifier struct{}
// Close is a no-op implementation of GCNotifier Close method.
func (n *nopGCNotifier) Close() {}
// AfterGC is a no-op implementation of GCNotifier AfterGC method.
func (c *nopGCNotifier) AfterGC() <-chan struct{} {
return nil
}

44
gcnotify/gcnotify.go Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gcnotify
import (
"github.com/CAFxX/gcnotifier"
"github.com/pilosa/pilosa"
)
// Ensure ActiveGCNotifier implements interface.
var _ pilosa.GCNotifier = &ActiveGCNotifier{}
type ActiveGCNotifier struct {
gcn *gcnotifier.GCNotifier
}
// NewActiveGCNotifier creates an active GCNotifier.
func NewActiveGCNotifier() *ActiveGCNotifier {
return &ActiveGCNotifier{
gcn: gcnotifier.New(),
}
}
// Close implements the GCNotifier interface.
func (n *ActiveGCNotifier) Close() {
n.gcn.Close()
}
// AfterGC implements the GCNotifier interface.
func (n *ActiveGCNotifier) AfterGC() <-chan struct{} {
return n.gcn.AfterGC()
}

115
gopsutil/systeminfo.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil
import (
"github.com/pilosa/pilosa"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
)
var _ pilosa.SystemInfo = NewSystemInfo()
// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS.
type SystemInfo struct {
platform string
family string
osVersion string
}
// Uptime returns the system uptime in seconds.
func (s *SystemInfo) Uptime() (uptime uint64, err error) {
hostInfo, err := host.Info()
if err != nil {
return 0, err
}
return hostInfo.Uptime, nil
}
// collectPlatformInfo fetches and caches system platform information.
func (s *SystemInfo) collectPlatformInfo() error {
var err error
if s.platform == "" {
s.platform, s.family, s.osVersion, err = host.PlatformInformation()
if err != nil {
return err
}
}
return nil
}
// Platform returns the system platform.
func (s *SystemInfo) Platform() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.platform, nil
}
// Family returns the system family.
func (s *SystemInfo) Family() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.family, err
}
// OSVersion returns the OS Version.
func (s *SystemInfo) OSVersion() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.osVersion, err
}
// MemFree returns the amount of free memory in bytes.
func (s *SystemInfo) MemFree() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Free, err
}
// MemTotal returns the amount of total memory in bytes.
func (s *SystemInfo) MemTotal() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Total, err
}
// MemUsed returns the amount of used memory in bytes.
func (s *SystemInfo) MemUsed() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Used, err
}
// KernelVersion returns the kernel version as a string.
func (s *SystemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *SystemInfo {
return &SystemInfo{}
}

77
gopsutil/systeminfo_test.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil_test
import (
"log"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gopsutil"
)
func TestSystemInfo(t *testing.T) {
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
// Uptime()(uint64, error)
// Platform()(string, error)
// Family()(string, error)
// OSVersion()(string, error)
// KernelVersion()(string, error)
// MemFree()(uint64, error)
// MemTotal()(uint64, error)
// MemUsed()(uint64, error)
//
uptime, err := systemInfo.Uptime()
if err != nil || uptime == 0 {
t.Fatalf("Error collecting uptime (error: %v)", err)
}
platform, err := systemInfo.Platform()
if err != nil {
t.Fatalf("Error getting platform. (platform: %v, error: %v)", platform, err)
}
family, err := systemInfo.Family()
if err != nil {
t.Fatalf("Error getting OS family. (family: %v, error: %v)", family, err)
}
osversion, err := systemInfo.OSVersion()
if err != nil {
t.Fatalf("Error getting OS version. (osversion: %v, error: %v)", osversion, err)
}
kernelversion, err := systemInfo.KernelVersion()
if err != nil {
t.Fatalf("Error getting kernel version. (kernelversion: %v, error: %v)", kernelversion, err)
}
memfree, err := systemInfo.MemFree()
if err != nil {
t.Fatalf("Error getting memfree. (memfree: %v, error: %v)", memfree, err)
}
memused, err := systemInfo.MemUsed()
if err != nil {
t.Fatalf("Error getting memused. (memused: %v, error: %v)", memused, err)
}
memtotal, err := systemInfo.MemTotal()
log.Println(memtotal)
if err != nil {
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
}
}

542
gossip/gossip.go
View file

@ -17,11 +17,16 @@ package gossip
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
@ -29,15 +34,15 @@ import (
"github.com/pkg/errors"
)
// Ensure GossipNodeSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipNodeSet{}
var _ pilosa.Gossiper = &GossipNodeSet{}
var _ memberlist.Delegate = &GossipNodeSet{}
// Ensure GossipMemberSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipMemberSet{}
var _ pilosa.Gossiper = &GossipMemberSet{}
var _ memberlist.Delegate = &GossipMemberSet{}
// GossipNodeSet represents a gossip implementation of NodeSet using memberlist
// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster
// GossipNodeSet also represents an implementation of memberlist.Delegate
type GossipNodeSet struct {
// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
type GossipMemberSet struct {
mu sync.RWMutex
node *pilosa.Node
memberlist *memberlist.Memberlist
handler pilosa.BroadcastHandler
@ -50,46 +55,60 @@ type GossipNodeSet struct {
LogOutput io.Writer
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (g *GossipNodeSet) Nodes() []*pilosa.Node {
a := make([]*pilosa.Node, 0, g.memberlist.NumMembers())
for _, n := range g.memberlist.Members() {
a = append(a, &pilosa.Node{Scheme: "gossip", Host: n.Name})
}
return a
}
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler
func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error {
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler.
func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error {
g.handler = h
return nil
}
// Seed returns the gossipSeed determined by the config.
func (g *GossipNodeSet) Seed() string {
return g.config.gossipSeed
// GetBindAddr returns the gossip bind address based on config and auto bind port.
// This method is currently only used in a test scenario where a second node needs
// the auto-bind address of the first node to use as its gossip seed.
func (g *GossipMemberSet) GetBindAddr() string {
return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)
}
// Open implements the NodeSet interface to start network activity.
func (g *GossipNodeSet) Open() error {
// Open implements the MemberSet interface to start network activity.
func (g *GossipMemberSet) Open(n *pilosa.Node) error {
if g.handler == nil {
return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()")
return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()")
}
ml, err := memberlist.Create(g.config.memberlistConfig)
g.node = n
err := error(nil)
g.mu.Lock()
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
if err != nil {
return errors.Wrap(err, "creating memberlist")
}
g.memberlist = ml
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
g.mu.RLock()
defer g.mu.RUnlock()
return g.memberlist.NumMembers()
},
RetransmitMult: 3,
}
// attach to gossip seed node
nodes := []*pilosa.Node{&pilosa.Node{Scheme: "gossip", Host: g.config.gossipSeed}} //TODO: support a list of seeds
err = g.joinWithRetry(pilosa.Nodes(nodes).Hosts())
var uris = make([]*pilosa.URI, len(g.config.gossipSeeds))
for i, addr := range g.config.gossipSeeds {
uris[i], err = pilosa.NewURIFromAddress(addr)
if err != nil {
return fmt.Errorf("new uri from address: %s", err)
}
}
var nodes = make([]*pilosa.Node, len(uris))
for i, uri := range uris {
nodes[i] = &pilosa.Node{URI: *uri}
}
g.mu.RLock()
err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings())
g.mu.RUnlock()
if err != nil {
return errors.Wrap(err, "joinWithRetry")
}
@ -97,7 +116,7 @@ func (g *GossipNodeSet) Open() error {
}
// joinWithRetry wraps the standard memberlist Join function in a retry.
func (g *GossipNodeSet) joinWithRetry(hosts []string) error {
func (g *GossipMemberSet) joinWithRetry(hosts []string) error {
err := retry(60, 2*time.Second, func() error {
_, err := g.memberlist.Join(hosts)
return err
@ -121,21 +140,334 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) {
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
// logger returns a logger for the GossipNodeSet.
func (g *GossipNodeSet) logger() *log.Logger {
// logger returns a logger for the GossipMemberSet.
func (g *GossipMemberSet) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
////////////////////////////////////////////////////////////////
type gossipConfig struct {
gossipSeed string
gossipSeeds []string
memberlistConfig *memberlist.Config
}
// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport.
func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) {
g := &GossipMemberSet{
LogOutput: server.LogOutput,
}
port := transport.Net.GetAutoBindPort()
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err)
}
host := bindURI.Host()
var gossipKey []byte
if cfg.Gossip.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key)
if err != nil {
return nil, fmt.Errorf("reading gossip key: %s", err)
}
}
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = transport.Net
conf.Name = name
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout)
conf.SuspicionMult = cfg.Gossip.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval)
conf.GossipNodes = cfg.Gossip.Nodes
conf.GossipInterval = time.Duration(cfg.Gossip.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime)
//
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeeds: cfg.Gossip.Seeds,
}
g.statusHandler = server
return g, nil
}
// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) {
port, err := strconv.Atoi(cfg.Gossip.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address: %s", err)
}
host := bindURI.Host()
// Set up the transport.
transport, err := NewTransport(host, port)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
return NewGossipMemberSetWithTransport(name, cfg, transport, server)
}
// SendSync implementation of the Broadcaster interface.
func (g *GossipMemberSet) SendSync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
mlist := g.memberlist
// Direct sends the message directly to every node.
// An error from any node raises an error on the entire operation.
//
// Gossip uses the gossip protocol to eventually deliver the message
// to every node.
var eg errgroup.Group
for _, n := range mlist.Members() {
// Don't send the message to the local node.
if n == mlist.LocalNode() {
continue
}
node := n
eg.Go(func() error {
return mlist.SendToTCP(node, msg)
})
}
return eg.Wait()
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
b := &broadcast{
msg: msg,
notify: nil,
}
g.broadcasts.QueueBroadcast(b)
return nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
if err != nil {
g.logger().Printf("marshal message error: %s", err)
return []byte{}
}
return buf
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipMemberSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
return
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side's LocalState.
func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
}
}
// GossipEventReceiver is used to enable an application to receive
// events about joins and leaves over a channel.
//
// Care must be taken that events are processed in a timely manner from
// the channel, since this delegate will block until an event can be sent.
type GossipEventReceiver struct {
ch chan memberlist.NodeEvent
eventHandler pilosa.EventHandler
// The writer for any logging.
LogOutput io.Writer
}
// NewGossipEventReceiver returns a new instance of GossipEventReceiver.
func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver {
return &GossipEventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
LogOutput: logOutput,
}
}
func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n}
}
func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n}
}
func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n}
}
// Start implements the pilosa.EventReceiver interface and sets the EventHandler.
func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error {
g.eventHandler = h
go g.listen()
return nil
}
// logger returns a logger for the GossipEventReceiver.
func (g *GossipEventReceiver) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
func (g *GossipEventReceiver) listen() {
var nodeEventType pilosa.NodeEventType
for {
e := <-g.ch
switch e.Event {
case memberlist.NodeJoin:
nodeEventType = pilosa.NodeJoin
case memberlist.NodeLeave:
nodeEventType = pilosa.NodeLeave
case memberlist.NodeUpdate:
nodeEventType = pilosa.NodeUpdate
default:
continue
}
// Get the node from the event.Node meta data.
var n internal.Node
if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta data")
}
node := pilosa.DecodeNode(&n)
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: node,
}
if err := g.eventHandler.ReceiveEvent(ne); err != nil {
g.logger().Printf("receive event error: %s", err)
}
}
}
// broadcast represents an implementation of memberlist.Broadcast
type broadcast struct {
msg []byte
notify chan<- struct{}
}
func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
return false
}
func (b *broadcast) Message() []byte {
return b.msg
}
func (b *broadcast) Finished() {
if b.notify != nil {
close(b.notify)
}
}
// Transport is a gossip transport for binding to a port.
type Transport struct {
//memberlist.Transport
Net *memberlist.NetTransport
URI *pilosa.URI
}
// NewTransport returns a NetTransport based on the given host and port.
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifiying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
net, err := newTransport(conf)
if err != nil {
return nil, fmt.Errorf("new transport: %s", err)
}
uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort()))
if err != nil {
return nil, fmt.Errorf("new uri from host port: %s", err)
}
return &Transport{
Net: net,
URI: uri,
}, nil
}
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
// This is useful for test cases where specifiying a port is not reasonable.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if conf.LogOutput != nil && conf.Logger != nil {
return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.")
@ -189,144 +521,6 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if err != nil {
return nil, fmt.Errorf("Could not set up network transport: %v", err)
}
if conf.BindPort == 0 {
port := nt.GetAutoBindPort()
conf.BindPort = port
conf.AdvertisePort = port
logger.Printf("[DEBUG] Using dynamic bind port %d", port)
}
return nt, nil
}
// NewGossipNodeSet returns a new instance of GossipNodeSet.
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipNodeSet, error) {
g := &GossipNodeSet{
LogOutput: server.LogOutput,
}
conf := memberlist.DefaultWANConfig()
conf.BindPort = gossipPort
conf.AdvertisePort = gossipPort
//TODO: pull memberlist config from pilosa.cfg file
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeed: gossipSeed,
}
g.config.memberlistConfig.Name = name
g.config.memberlistConfig.BindAddr = gossipHost
g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost)
g.config.memberlistConfig.Delegate = g
g.config.memberlistConfig.SecretKey = secretKey
g.statusHandler = server
// set up the transport
transport, err := newTransport(g.config.memberlistConfig)
if err != nil {
return nil, err
}
g.config.memberlistConfig.Transport = transport
// If no gossipSeed is provided, use local host:port.
if gossipSeed == "" {
g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, g.config.memberlistConfig.BindPort)
}
return g, nil
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipNodeSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return err
}
b := &broadcast{
msg: msg,
notify: nil,
}
g.broadcasts.QueueBroadcast(b)
return nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipNodeSet) NodeMeta(limit int) []byte {
return []byte{}
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipNodeSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
return
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipNodeSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side side's LocalState.
func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
}
}
// broadcast represents an implementation of memberlist.Broadcast
type broadcast struct {
msg []byte
notify chan<- struct{}
}
func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
return false
}
func (b *broadcast) Message() []byte {
return b.msg
}
func (b *broadcast) Finished() {
if b.notify != nil {
close(b.notify)
}
}

428
handler.go
View file

@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:generate statik -src=./webui
package pilosa
import (
@ -27,6 +25,7 @@ import (
"io/ioutil"
"log"
"net/http"
"net/url"
// Imported for its side-effect of registering pprof endpoints with the server.
_ "net/http/pprof"
"os"
@ -43,10 +42,6 @@ import (
"github.com/pilosa/pilosa/pql"
"unicode"
// Allow building Pilosa without the web UI.
_ "github.com/pilosa/pilosa/statik"
"github.com/rakyll/statik/fs"
)
// Handler represents an HTTP handler.
@ -56,12 +51,16 @@ type Handler struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
FileSystem FileSystem
// Local hostname & cluster configuration.
URI *URI
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Router *mux.Router
Router *mux.Router
NormalRouter *mux.Router
RestrictedRouter *mux.Router
// The execution engine for running queries.
Executor interface {
@ -70,6 +69,9 @@ type Handler struct {
// The writer for any logging.
LogOutput io.Writer
// Keeps the query argument validators for each handler
validators map[string]*queryValidationSpec
}
// externalPrefixFlag denotes endpoints that are intended to be exposed to clients.
@ -85,28 +87,117 @@ var externalPrefixFlag = map[string]bool{
"version": true,
}
type errorResponse struct {
Error string `json:"error"`
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler() *Handler {
handler := &Handler{
Broadcaster: NopBroadcaster,
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
FileSystem: NopFileSystem,
LogOutput: os.Stderr,
}
handler.Router = NewRouter(handler)
BuildRouters(handler)
handler.populateValidators()
return handler
}
// NewRouter creates a Gorilla Mux http router.
func NewRouter(handler *Handler) *mux.Router {
// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints.
func BuildRouters(handler *Handler) {
router := mux.NewRouter()
loadCommon(router, handler)
loadNormal(router, handler)
handler.NormalRouter = router
router.Use(handler.queryArgValidator)
// Restricted router.
router = mux.NewRouter()
loadCommon(router, handler)
loadRestricted(router, handler)
handler.RestrictedRouter = router
router.Use(handler.queryArgValidator)
handler.SetRestricted()
}
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index")
h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse")
h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits")
h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentBlocks"] = QueryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFrameRestore"] = QueryValidationSpecRequired("host")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
key := mux.CurrentRoute(r).GetName()
if validator, ok := h.validators[key]; ok {
if err := validator.validate(r.URL.Query()); err != nil {
// TODO: Return the response depending on the Accept header
response := errorResponse{Error: err.Error()}
body, err := json.Marshal(response)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
http.Error(w, string(body), http.StatusBadRequest)
return
}
}
next.ServeHTTP(w, r)
})
}
// SetNormal is a method of the SecurityManager interface which provides normal URI routing.
func (h *Handler) SetNormal() {
h.Router = h.NormalRouter
}
// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing.
func (h *Handler) SetRestricted() {
h.Router = h.RestrictedRouter
}
func loadCommon(router *mux.Router, handler *Handler) {
router.HandleFunc("/", handler.handleWebUI).Methods("GET")
router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET")
router.HandleFunc("/id", handler.handleGetID).Methods("GET")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.Use(handler.queryArgValidator)
}
func loadRestricted(router *mux.Router, handler *Handler) {
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted)
router.Use(handler.queryArgValidator)
}
func loadNormal(router *mux.Router, handler *Handler) {
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET")
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST")
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET")
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData")
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
@ -118,7 +209,7 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore")
router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
@ -129,16 +220,9 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH")
router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/id", handler.handleGetID).Methods("GET")
// TODO: Apply MethodNotAllowed statuses to all endpoints.
// Ideally this would be automatic, as described in this (wontfix) ticket:
@ -146,7 +230,10 @@ func NewRouter(handler *Handler) *mux.Router {
// For now we just do it for the most commonly used handler, /query
router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
return router
}
func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) {
http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.Cluster.State()), http.StatusMethodNotAllowed)
}
func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) {
@ -199,13 +286,13 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound)
return
}
statikFS, err := fs.New()
filesystem, err := h.FileSystem.New()
if err != nil {
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
return
}
http.FileServer(statikFS).ServeHTTP(w, r)
http.FileServer(filesystem).ServeHTTP(w, r)
}
// handleGetSchema handles GET /schema requests.
@ -219,13 +306,16 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
// handleGetStatus handles GET /status requests.
func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
status, err := h.StatusHandler.ClusterStatus()
pb, err := h.StatusHandler.ClusterStatus()
if err != nil {
h.logger().Printf("cluster status error: %s", err)
return
}
cs := pb.(*internal.ClusterStatus)
if err := json.NewEncoder(w).Encode(getStatusResponse{
Status: status,
State: cs.State,
Nodes: DecodeNodes(cs.Nodes),
}); err != nil {
h.logger().Printf("write status response error: %s", err)
}
@ -236,7 +326,8 @@ type getSchemaResponse struct {
}
type getStatusResponse struct {
Status proto.Message `json:"status"`
State string `json:"state"`
Nodes []*Node `json:"nodes"`
}
// handlePostQuery handles /query requests.
@ -308,31 +399,19 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
}
func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) {
var ms map[string]uint64
if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse {
ms = h.Holder.MaxInverseSlices()
} else {
ms = h.Holder.MaxSlices()
// handleGetSlicesMax handles GET /schema requests.
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.Holder.MaxSlices(),
Inverse: h.Holder.MaxInverseSlices(),
}); err != nil {
h.logger().Printf("write slices-max response error: %s", err)
}
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
pb := &internal.MaxSlicesResponse{
MaxSlices: ms,
}
if buf, err := proto.Marshal(pb); err != nil {
h.logger().Printf("protobuf marshal error: %s", err)
} else if _, err := w.Write(buf); err != nil {
h.logger().Printf("stream write error: %s", err)
}
return
}
json.NewEncoder(w).Encode(sliceMaxResponse{
MaxSlices: ms,
})
}
type sliceMaxResponse struct {
MaxSlices map[string]uint64 `json:"maxSlices"`
type getSlicesMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
Inverse map[string]uint64 `json:"inverse"`
}
// handleGetIndexes handles GET /index request.
@ -648,7 +727,6 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
}
h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
}
type _postFrameRequest postFrameRequest
@ -800,17 +878,30 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) {
return
}
// Create new field.
if err := f.CreateField(&Field{
field := &Field{
Name: fieldName,
Type: req.Type,
Min: req.Min,
Max: req.Max,
}); err != nil {
}
// Create new field.
if err := f.CreateField(field); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Send the create field message to all nodes.
err := h.Broadcaster.SendSync(
&internal.CreateFieldMessage{
Index: indexName,
Frame: frameName,
Field: encodeField(field),
})
if err != nil {
h.logger().Printf("problem sending CreateField message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
@ -844,6 +935,17 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request)
return
}
// Send the delete field message to all nodes.
err := h.Broadcaster.SendSync(
&internal.DeleteFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
})
if err != nil {
h.logger().Printf("problem sending DeleteField message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
@ -866,7 +968,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
return
}
schema, err := frame.GetFields()
fields, err := frame.GetFields()
if err == ErrFrameFieldsNotAllowed {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -876,7 +978,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
}
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: schema.Fields}); err != nil {
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
@ -929,7 +1031,7 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
// Delete the view.
if err := f.DeleteView(viewName); err != nil {
// Ingore this error becuase views do not exist on all nodes due to slice distribution.
// Ingore this error because views do not exist on all nodes due to slice distribution.
if err != ErrInvalidView {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -1070,12 +1172,6 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro
// readURLQueryRequest parses query parameters from URL parameters from r.
func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
q := r.URL.Query()
validQuery := validOptions(QueryRequest{})
for key := range q {
if _, ok := validQuery[key]; !ok {
return nil, errors.New("invalid query params")
}
}
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
@ -1173,9 +1269,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1243,9 +1339,9 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1309,9 +1405,9 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), index, slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, index, slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, index, slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1347,7 +1443,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
// Read slice parameter.
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest)
return
}
@ -1360,7 +1456,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentBackup handles GET /fragment/data requests.
// handleGetFragmentData handles GET /fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
@ -1383,7 +1479,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
}
}
// handlePostFragmentRestore handles POST /fragment/data requests.
// handlePostFragmentData handles POST /fragment/data requests.
func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
@ -1421,7 +1517,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentData handles GET /fragment/block/data requests.
// handleGetFragmentBlockData handles GET /fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
// Read request object.
var req internal.BlockDataRequest
@ -1537,7 +1633,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
// Loop over each slice and import it if this node owns it.
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
// Ignore this slice if we don't own it.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), indexName, slice) {
if !h.Cluster.OwnsFragment(h.Node.ID, indexName, slice) {
continue
}
@ -1921,6 +2017,131 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) {
}
}
// handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request.
func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req setCoordinatorRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
oldNode := h.Cluster.nodeByID(h.Cluster.Coordinator)
newNode := h.Cluster.nodeByID(req.ID)
if newNode == nil {
http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest)
return
}
if err := func() error {
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == h.Node.ID {
return h.Cluster.SetCoordinator(newNode)
}
// Send the set-coordinator message to new node.
err := h.Broadcaster.SendTo(
newNode,
&internal.SetCoordinatorMessage{
New: EncodeNode(newNode),
})
if err != nil {
return fmt.Errorf("problem sending SetCoordinator message: %s", err)
}
return nil
}(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(setCoordinatorResponse{
Old: oldNode,
New: newNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type setCoordinatorRequest struct {
ID string `json:"id"`
}
type setCoordinatorResponse struct {
Old *Node `json:"old"`
New *Node `json:"new"`
}
// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request.
func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req removeNodeRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
removeNode := h.Cluster.nodeByID(req.ID)
if removeNode == nil {
http.Error(w, fmt.Sprintf("Node is not a member of the cluster: %s", req.ID), http.StatusBadRequest)
return
}
// Start the resize process (similar to NodeJoin)
err = h.Cluster.NodeLeave(removeNode)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(removeNodeResponse{
Remove: removeNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type removeNodeRequest struct {
ID string `json:"id"`
}
type removeNodeResponse struct {
Remove *Node `json:"remove"`
}
// handlePostClusterResizeAbort handles POST /cluster/resize/abort request.
func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) {
var msg string
if err := func() error {
if !h.Cluster.IsCoordinator() {
return fmt.Errorf("abort requests must be made on the coordinator node")
}
err := h.Cluster.CompleteCurrentJob(ResizeJobStateAborted)
if err != nil {
return err
}
return nil
}(); err != nil {
msg = err.Error()
}
// Encode response.
if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{
Info: msg,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type clusterResizeAbortResponse struct {
Info string `json:"info"`
}
// InputJSONDataParser validates input json file and executes SetBit.
func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) {
inputDef, err := index.InputDefinition(name)
@ -1991,6 +2212,12 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
}
func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) {
err := h.Broadcaster.SendSync(&internal.RecalculateCaches{})
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
return
}
h.Holder.RecalculateCaches()
w.WriteHeader(http.StatusNoContent)
}
@ -2018,7 +2245,6 @@ func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error)
func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
fmt.Println("**unsupported media type**")
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
}
@ -2050,10 +2276,48 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
}
func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(h.Holder.LocalID))
_, err := w.Write([]byte(h.Cluster.Node.ID))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
type defaultClusterMessageResponse struct{}
type queryValidationSpec struct {
required []string
args map[string]struct{}
}
func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec {
args := map[string]struct{}{}
for _, arg := range requiredArgs {
args[arg] = struct{}{}
}
return &queryValidationSpec{
required: requiredArgs,
args: args,
}
}
func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec {
for _, arg := range args {
s.args[arg] = struct{}{}
}
return s
}
func (s queryValidationSpec) validate(query url.Values) error {
for _, req := range s.required {
if query.Get(req) == "" {
return errors.New(fmt.Sprintf("%s is required", req))
}
}
for k, _ := range query {
if _, ok := s.args[k]; !ok {
return errors.New(fmt.Sprintf("%s is not a valid argument", k))
}
}
return nil
}

73
handler_test.go
View file

@ -31,6 +31,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/test"
)
@ -106,6 +107,7 @@ func TestHandler_Schema(t *testing.T) {
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -146,11 +148,30 @@ func TestHandler_Status(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" {
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.SetRestricted()
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
}
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
@ -171,7 +192,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -212,7 +233,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -306,7 +327,7 @@ func TestHandler_Query_Params_Err(t *testing.T) {
test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
@ -1055,9 +1076,6 @@ func TestHandler_Frame_GetFields(t *testing.T) {
t.Fatal(err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f/fields")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
@ -1195,9 +1213,18 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler can return expvars without panicking.
@ -1819,3 +1846,31 @@ func TestHandler_RecalculateCaches(t *testing.T) {
}
}
func TestHandler_WebUI(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
if !strings.Contains(w.Body.String(), "<title>Pilosa WebUI</title>") {
t.Fatalf("WebUI is not being served correctly.")
}
// If curl is the client, the response should be different
w = httptest.NewRecorder()
req := test.MustNewHTTPRequest("GET", "/", nil)
req.Header.Add("User-Agent", "curl/7.54.0")
h.ServeHTTP(w, req)
if !strings.Contains(w.Body.String(), "try the WebUI") {
t.Fatalf("WebUI is not being served correctly.")
}
}

232
holder.go
View file

@ -31,6 +31,7 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/internal"
uuid "github.com/satori/go.uuid"
)
@ -48,8 +49,15 @@ type Holder struct {
// Indexes by name.
indexes map[string]*Index
hasData bool
// opened channel is closed once Open() completes.
opened chan struct{}
Broadcaster Broadcaster
NewAttrStore func(string) AttrStore
// Close management
wg sync.WaitGroup
closing chan struct{}
@ -64,8 +72,6 @@ type Holder struct {
CacheFlushInterval time.Duration
LogOutput io.Writer
LocalID string
}
// NewHolder returns a new instance of Holder.
@ -74,19 +80,54 @@ func NewHolder() *Holder {
indexes: make(map[string]*Index),
closing: make(chan struct{}, 0),
opened: make(chan struct{}),
Broadcaster: NopBroadcaster,
Stats: NopStatsClient,
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
LogOutput: os.Stderr,
}
}
// Peek reads the root data directory for the holder
// without actually loading any data into memory.
// HasData is returned, and h.hasData is set.
func (h *Holder) Peek() bool {
h.logger().Printf("peek at holder path: %s", h.Path)
h.hasData = false
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
return false
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return false
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
h.hasData = true
break
}
return h.hasData
}
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.setFileLimit()
h.logger().Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return err
}
@ -124,14 +165,19 @@ func (h *Holder) Open() error {
}
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
}
h.mu.Lock()
h.indexes[index.Name()] = index
h.mu.Unlock()
}
h.logger().Printf("open holder: complete")
// Periodically flush cache.
h.wg.Add(1)
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
h.Stats.Open()
close(h.opened)
return nil
}
@ -151,6 +197,15 @@ func (h *Holder) Close() error {
return nil
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
func (h *Holder) HasData() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return h.hasData || len(h.indexes) > 0
}
// MaxSlices returns MaxSlice map for all indexes.
func (h *Holder) MaxSlices() map[string]uint64 {
a := make(map[string]uint64)
@ -169,13 +224,13 @@ func (h *Holder) MaxInverseSlices() map[string]uint64 {
return a
}
// Schema returns schema data for all indexes and frames.
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name()}
fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
}
@ -189,6 +244,50 @@ func (h *Holder) Schema() []*IndexInfo {
return a
}
// ApplySchema applies an internal Schema to Holder.
func (h *Holder) ApplySchema(schema *internal.Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
opt := IndexOptions{}
idx, err := h.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return err
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
if err != nil {
return err
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
if err != nil {
return err
}
}
}
// TODO: Create inputDefinitions that don't exist.
}
return nil
}
// EncodeMaxSlices creates and internal representation of max slices.
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}
// EncodeSchema creates an internal representation of schema.
func (h *Holder) EncodeSchema() *internal.Schema {
return &internal.Schema{
Indexes: EncodeIndexes(h.Indexes()),
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -278,6 +377,8 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
index.LogOutput = h.LogOutput
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.Broadcaster
index.NewAttrStore = h.NewAttrStore
index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data"))
return index, nil
}
@ -432,22 +533,29 @@ func (h *Holder) setFileLimit() {
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
func (h *Holder) loadLocalID() error {
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, "ID")
localID := ""
localIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
localID = strings.TrimSpace(string(localIDBytes))
} else {
u := uuid.NewV4()
localID = u.String()
err = ioutil.WriteFile(idPath, []byte(localID), 0600)
if err != nil {
return err
}
nodeID := ""
h.logger().Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return "", err
}
h.LocalID = localID
return nil
nodeIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
nodeID = strings.TrimSpace(string(nodeIDBytes))
} else if os.IsNotExist(err) {
nodeID = uuid.NewV4().String()
err = ioutil.WriteFile(idPath, []byte(nodeID), 0600)
if err != nil {
return "", err
}
} else if err != nil {
return "", err
}
return nodeID, nil
}
// HolderSyncer is an active anti-entropy tool that compares the local holder
@ -455,7 +563,7 @@ func (h *Holder) loadLocalID() error {
type HolderSyncer struct {
Holder *Holder
URI *URI
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -511,7 +619,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.URI.HostPort(), di.Name, slice) {
if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) {
continue
}
@ -553,11 +661,8 @@ func (s *HolderSyncer) syncIndex(index string) error {
s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
@ -567,7 +672,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.Host})
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID})
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
@ -586,7 +691,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve index reference.
// Retrieve frame reference.
f := s.Holder.Frame(index, name)
if f == nil {
return nil
@ -602,11 +707,8 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
@ -618,7 +720,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.Host})
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@ -658,7 +760,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Host: s.URI.HostPort(),
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
@ -669,3 +771,65 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
return nil
}
// HolderCleaner removes fragments and data files that are no longer used.
type HolderCleaner struct {
Node *Node
Holder *Holder
Cluster *Cluster
// Signals that the sync should stop.
Closing <-chan struct{}
}
// IsClosing returns true if the cleaner has been marked to close.
func (c *HolderCleaner) IsClosing() bool {
select {
case <-c.Closing:
return true
default:
return false
}
}
// CleanHolder compares the holder with the cluster state and removes
// any unnecessary fragments and files.
func (c *HolderCleaner) CleanHolder() error {
for _, index := range c.Holder.Indexes() {
// Verify cleaner has not closed.
if c.IsClosing() {
return nil
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
continue
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
return err
}
}
}
}
}
return nil
}
func uint64InSlice(i uint64, s []uint64) bool {
for _, o := range s {
if i == o {
return true
}
}
return false
}

211
holder_test.go
View file

@ -284,6 +284,60 @@ func TestHolder_Open(t *testing.T) {
})
}
func TestHolder_HasData(t *testing.T) {
t.Run("IndexDirectory", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if h.HasData() {
t.Fatal("expected HasData to return false")
}
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if !h.HasData() {
t.Fatal("expected HasData to return true")
}
})
t.Run("Peek", func(t *testing.T) {
h := test.NewHolder()
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
}
// Create an index directory to indicate data exists.
if err := os.Mkdir(h.IndexPath("test"), 0777); err != nil {
t.Fatal(err)
}
if hasData := h.Peek(); hasData != true {
t.Fatal("expected Peek to return true")
} else if !h.HasData() {
t.Fatal("expected HasData to return true")
}
})
t.Run("Peek at missing directory", func(t *testing.T) {
h := test.NewHolder()
// Ensure that hasData is false when trying to peek into
// a directory that doesn't exist.
h.Path = "bad-path"
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
}
})
}
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
@ -334,16 +388,21 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(client)
e.Holder = hldr1.Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].Host = "localhost:0"
cluster.Nodes[1].Host = test.MustParseURLHost(s.URL)
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
t.Fatal(err)
}
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0, hldr1} {
@ -395,13 +454,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr0.Index("y").SetRemoteMaxSlice(3)
// Set up syncer.
uri, err := cluster.Nodes[0].URI()
if err != nil {
t.Fatal(err)
}
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
URI: uri,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: pilosa.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
@ -440,3 +495,141 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
}
}
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
t.Fatal(err)
}
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(2)
// Keep replication the same and ensure we get the expected results.
cluster.ReplicaN = 2
// Set up cleaner for replication 2.
cleaner2 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner2.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
// Change replication factor to ensure we have fragments to remove.
cluster.ReplicaN = 1
// Set up cleaner for replication 1.
cleaner1 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner1.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
if f != nil {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
}

54
index.go
View file

@ -55,8 +55,10 @@ type Index struct {
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
NewAttrStore func(string) AttrStore
// Column attribute storage and cache.
columnAttrStore *AttrStore
columnAttrStore AttrStore
// InputDefinitions by name.
inputDefinitions map[string]*InputDefinition
@ -83,7 +85,8 @@ func NewIndex(path, name string) (*Index, error) {
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
columnLabel: DefaultColumnLabel,
@ -100,7 +103,7 @@ func (i *Index) Name() string { return i.name }
func (i *Index) Path() string { return i.path }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore }
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (i *Index) SetColumnLabel(v string) error {
@ -256,9 +259,7 @@ func (i *Index) Close() error {
defer i.mu.Unlock()
// Close the attribute store.
if i.columnAttrStore != nil {
i.columnAttrStore.Close()
}
i.columnAttrStore.Close()
// Close all frames.
for _, f := range i.frames {
@ -392,6 +393,20 @@ func (i *Index) Frames() []*Frame {
return a
}
// InputDefinitions returns a list of all inputDefinitions in the index.
func (i *Index) InputDefinitions() []*InputDefinition {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*InputDefinition, 0, len(i.inputDefinitions))
for _, d := range i.inputDefinitions {
a = append(a, d)
}
//sort.Sort(inputDefintionSlice(a)) // TODO
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
@ -440,8 +455,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
if opt.RangeEnabled {
if opt.InverseEnabled {
return nil, ErrInverseRangeNotAllowed
} else if opt.CacheType != "" && opt.CacheType != CacheTypeNone {
return nil, ErrRangeCacheNotAllowed
}
} else {
if len(opt.Fields) > 0 {
@ -494,18 +507,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
f.inverseEnabled = opt.InverseEnabled
f.rangeEnabled = opt.RangeEnabled
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
f.rangeEnabled = opt.RangeEnabled
// Set schema & save.
f.schema = &FrameSchema{
Fields: opt.Fields,
}
if err := f.saveSchema(); err != nil {
// Set fields.
f.fields = opt.Fields
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
@ -524,6 +531,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) {
f.LogOutput = i.LogOutput
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
return f, nil
}
@ -623,12 +631,10 @@ func EncodeIndexes(a []*Index) []*internal.Index {
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
io := d.options()
return &internal.Index{
Name: d.name,
Meta: io.Encode(),
MaxSlice: d.MaxSlice(),
Frames: encodeFrames(d.Frames()),
Name: d.name,
Frames: encodeFrames(d.Frames()),
InputDefinitions: encodeInputDefinitions(d.InputDefinitions()),
}
}
@ -723,7 +729,6 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) {
if err != nil {
return nil, err
}
inputDef.broadcaster = i.broadcaster
return inputDef, nil
}
@ -776,7 +781,6 @@ func (i *Index) openInputDefinitions() error {
return nil
}
}
}
return nil
}

24
index_test.go
View file

@ -106,25 +106,21 @@ func TestIndex_CreateFrame(t *testing.T) {
},
}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema: %#v", f.Schema())
t.Fatalf("unexpected fields: %#v", f.Fields())
}
// Reopen the index & verify the fields are loaded.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema after reopen: %#v", f.Schema())
t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
}
})
@ -140,14 +136,14 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) {
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeRanked,
}); err != pilosa.ErrRangeCacheNotAllowed {
}); err != nil {
t.Fatal(err)
}
})

60
input_definition.go
View file

@ -36,12 +36,11 @@ var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleR
// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
name string
path string
index string
broadcaster Broadcaster
frames []InputFrame
fields []InputDefinitionField
name string
path string
index string
frames []InputFrame
fields []InputDefinitionField
}
// NewInputDefinition returns a new instance of InputDefinition.
@ -86,17 +85,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
// Copy metadata fields.
i.name = pb.Name
for _, fr := range pb.Frames {
frameMeta := fr.Meta
inputFrame := InputFrame{
Name: fr.Name,
Options: FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
RowLabel: DefaultRowLabel,
InverseEnabled: frameMeta.InverseEnabled,
CacheSize: frameMeta.CacheSize,
CacheType: frameMeta.CacheType,
TimeQuantum: TimeQuantum(frameMeta.TimeQuantum),
},
Name: fr.Name,
Options: *decodeFrameOptions(fr.Meta),
}
i.frames = append(i.frames, inputFrame)
}
@ -338,6 +329,43 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition {
return &def
}
// encodeInputDefinitions converts a into its internal representation.
func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition {
other := make([]*internal.InputDefinition, len(a))
for i := range a {
other[i] = encodeInputDefinition(a[i])
}
return other
}
// encodeInputDefinition converts i into its internal representation.
func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition {
//fo := f.options()
return &internal.InputDefinition{
Name: i.name,
Frames: encodeInputFrames(i.frames),
Fields: encodeInputDefinitionFields(i.fields),
}
}
// encodeInputFrames converts a into its internal representation.
func encodeInputFrames(a []InputFrame) []*internal.Frame {
other := make([]*internal.Frame, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// encodeInputDefinitionFields converts a into its internal representation.
func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField {
other := make([]*internal.InputDefinitionField, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// AddFrame manually add frame to input definition.
func (i *InputDefinition) AddFrame(frame InputFrame) error {
i.frames = append(i.frames, frame)

4179
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

106
internal/private.proto
View file

@ -38,8 +38,9 @@ message Cache {
repeated uint64 IDs = 1;
}
message MaxSlicesResponse {
map<string, uint64> MaxSlices = 1;
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
@ -68,17 +69,31 @@ message DeleteFrameMessage {
string Frame = 2;
}
message CreateFieldMessage {
string Index = 1;
string Frame = 2;
Field Field = 3;
}
message DeleteFieldMessage {
string Index = 1;
string Frame = 2;
string Field = 3;
}
message Frame {
string Name = 1;
FrameMeta Meta = 2;
repeated string Views = 3;
}
message Schema {
repeated Index Indexes = 1;
}
message Index {
string Name = 1;
IndexMeta Meta = 2;
uint64 MaxSlice = 3;
repeated Frame Frames = 4;
repeated uint64 Slices = 5;
repeated InputDefinition InputDefinitions = 6;
}
@ -112,19 +127,38 @@ message DeleteInputDefinitionMessage {
string Name = 2;
}
message NodeStatus {
string Host = 1;
message URI {
string Scheme = 1;
string Host = 2;
uint32 Port = 3;
}
message Node {
string ID = 1;
URI URI = 2;
bool IsCoordinator = 3;
}
message NodeStateMessage {
string NodeID = 1;
string State = 2;
repeated Index Indexes = 3;
string Scheme = 4;
}
message NodeEventMessage {
uint32 Event = 1;
Node Node = 2;
}
message NodeStatus {
Node Node = 1;
MaxSlices MaxSlices = 2;
Schema Schema = 3;
}
message ClusterStatus {
repeated NodeStatus Nodes = 1;
}
message FrameSchema {
repeated Field Fields = 1;
string ClusterID = 1;
string State = 2;
repeated Node Nodes = 3;
}
message Field {
@ -134,8 +168,52 @@ message Field {
int64 Max = 4;
}
message CreateViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message ResizeInstruction {
int64 JobID = 1;
Node Node = 2;
Node Coordinator = 3;
repeated ResizeSource Sources = 4;
Schema Schema = 5;
ClusterStatus ClusterStatus = 6;
}
message ResizeSource {
Node Node = 1;
string Index = 2;
string Frame = 3;
string View = 4;
uint64 Slice = 5;
}
message ResizeInstructionComplete {
int64 JobID = 1;
Node Node = 2;
string Error = 3;
}
message SetCoordinatorMessage {
Node New = 1;
}
message UpdateCoordinatorMessage {
Node New = 1;
}
message Topology {
string ClusterID = 1;
repeated string NodeIDs = 2;
}
message RecalculateCaches {}

47
pilosa.go
View file

@ -73,8 +73,7 @@ var (
ErrQueryRequired = errors.New("query required")
ErrTooManyWrites = errors.New("too many write commands")
ErrConfigClusterTypeInvalid = errors.New("invalid cluster type")
ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts")
ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed")
)
// Regular expression to validate index and frame names.
@ -162,6 +161,50 @@ func StringInSlice(a string, list []string) bool {
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {

68
roaring/assembly.go
View file

@ -1,68 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
// bit population count, take from
// https://code.google.com/p/go/issues/detail?id=4988#c11
// credit: https://code.google.com/u/arnehormann/
func popcntGo(x uint64) (n uint64) {
x -= (x >> 1) & 0x5555555555555555
x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
x += x >> 4
x &= 0x0f0f0f0f0f0f0f0f
x *= 0x0101010101010101
return x >> 56
}
func popcntSliceGo(s []uint64) uint64 {
cnt := uint64(0)
for _, x := range s {
cnt += popcntGo(x)
}
return cnt
}
func popcntMaskSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] &^ m[i])
}
return cnt
}
func popcntAndSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] & m[i])
}
return cnt
}
func popcntOrSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] | m[i])
}
return cnt
}
func popcntXorSliceGo(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcntGo(s[i] ^ m[i])
}
return cnt
}

122
roaring/assembly_amd64.s
View file

@ -1,122 +0,0 @@
#include "textflag.h"
TEXT ·hasAsm(SB),4,$0-1
MOVQ $1, AX
CPUID
SHRQ $23, CX
ANDQ $1, CX
MOVB CX, ret+0(FP)
RET
TEXT ·POPCNTQ(SB),NOSPLIT,$0-16
MOVQ memory+0(FP), BP
POPCNTQ BP, BX
MOVQ BX, ret+8(FP)
RET
TEXT ·BSFQ(SB),NOSPLIT,$0-16
MOVQ memory+0(FP), BP
BSFQ BP, BX
MOVQ BX, ret+8(FP)
RET
#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2
TEXT ·popcntSliceAsm(SB),4,$0-32
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntSliceEnd
popcntSliceLoop:
BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX
ADDQ DX, AX
ADDQ $8, SI
LOOP popcntSliceLoop
popcntSliceEnd:
MOVQ AX, ret+24(FP)
RET
TEXT ·popcntMaskSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntMaskSliceEnd
MOVQ m+24(FP), DI
popcntMaskSliceLoop:
MOVQ (DI), DX
NOTQ DX
ANDQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntMaskSliceLoop
popcntMaskSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntAndSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntAndSliceEnd
MOVQ m+24(FP), DI
popcntAndSliceLoop:
MOVQ (DI), DX
ANDQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntAndSliceLoop
popcntAndSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntOrSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntOrSliceEnd
MOVQ m+24(FP), DI
popcntOrSliceLoop:
MOVQ (DI), DX
ORQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntOrSliceLoop
popcntOrSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntXorSliceAsm(SB),4,$0-56
XORQ AX, AX
MOVQ s_base+0(FP), SI
MOVQ s_len+8(FP), CX
TESTQ CX, CX
JZ popcntXorSliceEnd
MOVQ m+24(FP), DI
popcntXorSliceLoop:
MOVQ (DI), DX
XORQ (SI), DX
POPCNTQ_DX_DX
ADDQ DX, AX
ADDQ $8, SI
ADDQ $8, DI
LOOP popcntXorSliceLoop
popcntXorSliceEnd:
MOVQ AX, ret+48(FP)
RET
TEXT ·popcntAsm(SB),4,$0-16
MOVQ x+0(FP), DX
POPCNTQ_DX_DX
MOVQ DX, ret+8(FP)
RET

87
roaring/assembly_asm.go
View file

@ -1,87 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build amd64
package roaring
func hasAsm() bool
func BSFQ(memory uint64) int
func POPCNTQ(memory uint64) int
//go:noescape
var useAsm = hasAsm()
//go:noescape
func popcntSliceAsm(s []uint64) uint64
//go:noescape
func popcntMaskSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntAndSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntOrSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntXorSliceAsm(s, m []uint64) uint64
//go:noescape
func popcntAsm(x uint64) uint64
func popcntSlice(s []uint64) uint64 {
if useAsm {
return popcntSliceAsm(s)
}
return popcntSliceGo(s)
}
func popcntMaskSlice(s, m []uint64) uint64 {
if useAsm {
return popcntMaskSliceAsm(s, m)
}
return popcntMaskSliceGo(s, m)
}
func popcntAndSlice(s, m []uint64) uint64 {
if useAsm {
return popcntAndSliceAsm(s, m)
}
return popcntAndSliceGo(s, m)
}
func popcntOrSlice(s, m []uint64) uint64 {
if useAsm {
return popcntOrSliceAsm(s, m)
}
return popcntOrSliceGo(s, m)
}
func popcntXorSlice(s, m []uint64) uint64 {
if useAsm {
return popcntXorSliceAsm(s, m)
}
return popcntXorSliceGo(s, m)
}
func popcnt(x uint64) uint64 {
if useAsm {
return popcntAsm(x)
}
return popcntGo(x)
}

139
roaring/assembly_test.go
View file

@ -1,139 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
import "testing"
func TestBSFQ(t *testing.T) {
result := BSFQ(2)
if result != 1 {
t.Fatalf("BSF INCORRECT: %d", result)
}
}
func TestBSFQ_CompareGo(t *testing.T) {
v := uint64(1)
for i := 0; i < 64; i++ {
if BSFQ(v) != trailingZeroN(v) {
t.Fatalf("BSF INCORRECT: %d %d", BSFQ(v), trailingZeroN(v))
}
if v == 0 {
v = 1
} else {
v *= 2
}
}
/*
if bsfq(0) != trailingZeroN(0) {
fmt.Println(bsfq(0))
t.Fatalf("BSF INCORRECT")
}
*/
}
func BenchmarkBSF(b *testing.B) {
for i := 0; i < b.N; i++ {
BSFQ(uint64(i))
}
}
func BenchmarkTrailingZeroN(b *testing.B) {
for i := 0; i < b.N; i++ {
trailingZeroN(uint64(i))
}
}
func BenchmarkPOPCNTQ(b *testing.B) {
for i := 0; i < b.N; i++ {
POPCNTQ(uint64(i))
}
}
func BenchmarkPopcount(b *testing.B) {
for i := 0; i < b.N; i++ {
popcount(uint64(i))
}
}
func BenchmarkPopcntAsm(b *testing.B) {
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
popcntAsm(0xdeadbeef)
}
}
func BenchmarkPopcntGo(b *testing.B) {
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
popcntGo(0xdeadbeef)
}
}
func getData() []uint64 {
return []uint64{
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
0xdeadbeef,
}
}
func BenchmarkPopcntSliceGo(b *testing.B) {
d := getData()
for n := 0; n < b.N; n++ {
popcntSliceGo(d)
}
}
func BenchmarkPopcntSliceAsm(b *testing.B) {
d := getData()
for n := 0; n < b.N; n++ {
popcntSliceAsm(d)
}
}
func BenchmarkPopcntSlice(b *testing.B) {
d := getData()
for n := 0; n < b.N; n++ {
popcntSlice(d)
}
}

154
roaring/roaring.go
View file

@ -21,6 +21,7 @@ import (
"fmt"
"hash/fnv"
"io"
"math/bits"
"sort"
"unsafe"
)
@ -1001,12 +1002,16 @@ const ArrayMaxSize = 4096
// RunMaxSize represents the maximum size of run length encoded containers.
const RunMaxSize = 2048
// container represents a container for uint32 integers.
// container represents a container for uint16 integers.
//
// These are used for storing the low bits. Containers are separated into three
// types depending on cardinality. For containers with less than 4,096 values,
// an array or RLE container is used, depending on the contents. For containers
// with more than 4,096 values, the values are encoded into bitmaps.
// These are used for storing the low bits of numbers in larger sets of uint64.
// The high bits are stored in a container's key which is tracked by a separate
// data structure. Integers in a container can be encoded in one of three ways -
// the encoding used is usually whichever is most compact, though any container
// type should be able to encode any set of integers safely. For containers with
// less than 4,096 values, an array is often used. Containers with long runs of
// integers would use run length encoding, and more random data usually uses
// bitmap encoding.
type container struct {
mapped bool // mapped directly to a byte slice when true
containerType byte // array, bitmap, or run
@ -1274,10 +1279,10 @@ func (c *container) contains(v uint16) bool {
func (c *container) bitmapCountRuns() (r int) {
for i := 0; i < 1023; i++ {
v, v1 := c.bitmap[i], c.bitmap[i+1]
r = r + int(popcnt((v<<1)&^v)+((v>>63)&^v1))
r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1))
}
vl := c.bitmap[len(c.bitmap)-1]
r = r + int(popcnt((vl<<1)&^vl)+vl>>63)
r = r + int(popcount((vl<<1)&^vl)+vl>>63)
return r
}
@ -1906,19 +1911,20 @@ func intersectionCountBitmapRun(a, b *container) (n int) {
}
func intersectionCountArrayBitmap(a, b *container) (n int) {
ln := len(b.bitmap)
for _, val := range a.array {
i := val >> 6
if i >= uint16(len(b.bitmap)) {
i := int(val >> 6)
if i >= ln {
break
}
off := val % 64
n += int((b.bitmap[i] & (1 << off)) >> off)
n += int(b.bitmap[i]>>off) & 1
}
return n
}
func intersectionCountBitmapBitmap(a, b *container) (n int) {
return int(popcntAndSlice(a.bitmap, b.bitmap))
return int(popcountAndSlice(a.bitmap, b.bitmap))
}
func intersect(a, b *container) *container {
@ -2061,22 +2067,22 @@ func intersectBitmapRun(a, b *container) *container {
for valast >= vb.start && vastart <= vb.last && i < bitmapN {
if vastart >= vb.start && valast <= vb.last { // a within b
output.bitmap[i] = a.bitmap[i]
output.n += int(popcnt(a.bitmap[i]))
output.n += int(popcount(a.bitmap[i]))
} else if vb.start >= vastart && vb.last <= valast { // b within a
var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
bits := a.bitmap[i] & mask
output.bitmap[i] |= bits
output.n += int(popcnt(bits))
output.n += int(popcount(bits))
} else if vastart < vb.start { // a overlaps front of b
offset := 64 - (1 + valast - vb.start)
bits := (a.bitmap[i] >> offset) << offset
output.bitmap[i] |= bits
output.n += int(popcnt(bits))
output.n += int(popcount(bits))
} else if vb.start < vastart { // b overlaps front of a
offset := 64 - (1 + vb.last - vastart)
bits := (a.bitmap[i] << offset) >> offset
output.bitmap[i] |= bits
output.n += int(popcnt(bits))
output.n += int(popcount(bits))
}
// update loop vars
i++
@ -2293,19 +2299,19 @@ func (c *container) bitmapSetRange(i, j uint64) {
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
xcnt := popcnt(X)
ycnt := popcnt(Y)
xcnt := popcount(X)
ycnt := popcount(Y)
if x == y {
c.n += int((j - i) - popcnt(c.bitmap[x]&(X&Y)))
c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y)))
c.bitmap[x] |= (X & Y)
} else {
c.n += int(xcnt - popcnt(c.bitmap[x]&X))
c.n += int(xcnt - popcount(c.bitmap[x]&X))
c.bitmap[x] |= X
for i := x + 1; i < y; i++ {
c.n += int(64 - popcnt(c.bitmap[i]))
c.n += int(64 - popcount(c.bitmap[i]))
c.bitmap[i] = maxBitmap
}
c.n += int(ycnt - popcnt(c.bitmap[y]&Y))
c.n += int(ycnt - popcount(c.bitmap[y]&Y))
c.bitmap[y] |= Y
}
}
@ -2317,21 +2323,21 @@ func (c *container) bitmapXorRange(i, j uint64) {
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
cnt := popcnt(c.bitmap[x])
cnt := popcount(c.bitmap[x])
c.bitmap[x] ^= (X & Y) //// flip
c.n += int(popcnt(c.bitmap[x]) - cnt)
c.n += int(popcount(c.bitmap[x]) - cnt)
} else {
cnt := popcnt(c.bitmap[x])
cnt := popcount(c.bitmap[x])
c.bitmap[x] ^= X
c.n += int(popcnt(c.bitmap[x]) - cnt)
c.n += int(popcount(c.bitmap[x]) - cnt)
for i := x + 1; i < y; i++ {
cnt = popcnt(c.bitmap[i])
cnt = popcount(c.bitmap[i])
c.bitmap[i] ^= maxBitmap
c.n += int(popcnt(c.bitmap[i]) - cnt)
c.n += int(popcount(c.bitmap[i]) - cnt)
}
cnt = popcnt(c.bitmap[y])
cnt = popcount(c.bitmap[y])
c.bitmap[y] ^= Y
c.n += int(popcnt(c.bitmap[y]) - cnt)
c.n += int(popcount(c.bitmap[y]) - cnt)
}
}
@ -2342,16 +2348,16 @@ func (c *container) bitmapZeroRange(i, j uint64) {
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
c.n -= int(popcnt(c.bitmap[x] & (X & Y)))
c.n -= int(popcount(c.bitmap[x] & (X & Y)))
c.bitmap[x] &= ^(X & Y)
} else {
c.n -= int(popcnt(c.bitmap[x] & X))
c.n -= int(popcount(c.bitmap[x] & X))
c.bitmap[x] &= ^X
for i := x + 1; i < y; i++ {
c.n -= int(popcnt(c.bitmap[i]))
c.n -= int(popcount(c.bitmap[i]))
c.bitmap[i] = 0
}
c.n -= int(popcnt(c.bitmap[y] & Y))
c.n -= int(popcount(c.bitmap[y] & Y))
c.bitmap[y] &= ^Y
}
}
@ -2376,7 +2382,7 @@ func unionBitmapBitmap(a, b *container) *container {
for i := 0; i < bitmapN; i++ {
v := a.bitmap[i] | b.bitmap[i]
output.bitmap[i] = v
output.n += int(popcnt(v))
output.n += int(popcount(v))
}
return output
@ -2640,7 +2646,7 @@ func differenceRunRun(a, b *container) *container {
for apos < alen && bpos < blen {
switch {
case alast < bstart:
// current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run
// current A-run entirely precedes current B-run: keep full A-run, advance to next A-run
output.runs = append(output.runs, interval16{start: astart, last: alast})
apos++
if apos < alen {
@ -2648,7 +2654,7 @@ func differenceRunRun(a, b *container) *container {
alast = a.runs[apos].last
}
case blast < astart:
// current B-run entirely preceeds current A-run: advance to next B-run
// current B-run entirely precedes current A-run: advance to next B-run
bpos++
if bpos < blen {
bstart = b.runs[bpos].start
@ -2812,7 +2818,7 @@ func xorBitmapBitmap(a, b *container) *container {
for i := 0; i < bitmapN; i++ {
v := a.bitmap[i] ^ b.bitmap[i]
output.bitmap[i] = v
output.n += int(popcnt(v))
output.n += int(popcount(v))
}
if output.count() < ArrayMaxSize {
@ -2975,35 +2981,7 @@ func search64(a []uint64, value uint64) int {
// trailingZeroN returns the number of trailing zeros in v.
// v must be greater than zero.
func trailingZeroN(v uint64) int {
n := int64(63)
if y := v << 32; y != 0 {
n, v = n-32, y
}
if y := v << 16; y != 0 {
n, v = n-16, y
}
if y := v << 8; y != 0 {
n, v = n-8, y
}
if y := v << 4; y != 0 {
n, v = n-4, y
}
if y := v << 2; y != 0 {
n, v = n-2, y
}
return int(n - int64(v<<1>>63))
}
// bit population count, taken from
// https://code.google.com/p/go/issues/detail?id=4988#c11
// credit: https://code.google.com/u/arnehormann/
func popcount(x uint64) (n uint64) {
x -= (x >> 1) & 0x5555555555555555
x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
x += x >> 4
x &= 0x0f0f0f0f0f0f0f0f
x *= 0x0101010101010101
return x >> 56
return bits.TrailingZeros64(v)
}
// ErrorList represents a list of errors.
@ -3265,3 +3243,47 @@ func xorBitmapRun(a, b *container) *container {
}
return output
}
func popcount(x uint64) uint64 {
return uint64(bits.OnesCount64(x))
}
func popcountSlice(s []uint64) uint64 {
cnt := uint64(0)
for _, x := range s {
cnt += popcount(x)
}
return cnt
}
func popcountMaskSlice(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcount(s[i] &^ m[i])
}
return cnt
}
func popcountAndSlice(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcount(s[i] & m[i])
}
return cnt
}
func popcountOrSlice(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcount(s[i] | m[i])
}
return cnt
}
func popcountXorSlice(s, m []uint64) uint64 {
cnt := uint64(0)
for i := range s {
cnt += popcount(s[i] ^ m[i])
}
return cnt
}

2
roaring/roaring_test.go
View file

@ -291,7 +291,7 @@ func TestBitmap_Max(t *testing.T) {
}
}
// Ensure CountRange is correct even if rangekey is prior to inital container.
// Ensure CountRange is correct even if rangekey is prior to initial container.
func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
s := uint64(2009 * 1048576)
e := uint64(2010 * 1048576)

23
roaring/assembly_generic.go → security_manager.go
View file

@ -12,16 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !amd64
package pilosa
package roaring
// SecurityManager provides the ability to limit access to restricted endpoints
// during cluster configuration.
type SecurityManager interface {
SetRestricted()
SetNormal()
}
func hasAsm() bool {return false}
// NopSecurityManager provides a no-op implementation of the SecurityManager interface.
type NopSecurityManager struct {
}
// SetRestricted no-op.
func (sdm *NopSecurityManager) SetRestricted() {}
func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) }
func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) }
func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) }
func popcntOrSlice(s, m []uint64) uint64 { return popcntOrSliceGo(s, m) }
func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) }
func popcnt(s uint64) uint64 { return popcntGo(s) }
// SetNormal no-op.
func (sdm *NopSecurityManager) SetNormal() {}

586
server.go
View file

@ -23,7 +23,6 @@ import (
"log"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"runtime"
@ -32,17 +31,15 @@ import (
"sync"
"time"
"github.com/CAFxX/gcnotifier"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/diagnostics"
"github.com/pilosa/pilosa/internal"
"golang.org/x/sync/errgroup"
)
// Default server settings.
const (
DefaultAntiEntropyInterval = 10 * time.Minute
DefaultPollingInterval = 60 * time.Second
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
)
@ -68,16 +65,19 @@ type Server struct {
RemoteClient *http.Client
// Cluster configuration.
// Host is replaced with actual host after opening if port is ":0".
Network string
URI *URI
NodeID string
URI URI
Cluster *Cluster
diagnostics *diagnostics.Diagnostics
ClusterID string
diagnostics *DiagnosticsCollector
SystemInfo SystemInfo
GCNotifier GCNotifier
NewAttrStore func(string) AttrStore
// Background monitoring intervals.
AntiEntropyInterval time.Duration
PollingInterval time.Duration
MetricInterval time.Duration
DiagnosticInterval time.Duration
@ -102,12 +102,16 @@ func NewServer() *Server {
Handler: NewHandler(),
Broadcaster: NopBroadcaster,
BroadcastReceiver: NopBroadcastReceiver,
diagnostics: diagnostics.New(DefaultDiagnosticServer),
diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
SystemInfo: NewNopSystemInfo(),
Network: "tcp",
GCNotifier: NopGCNotifier,
NewAttrStore: NewNopAttrStore,
AntiEntropyInterval: DefaultAntiEntropyInterval,
PollingInterval: DefaultPollingInterval,
MetricInterval: 0,
DiagnosticInterval: 0,
@ -116,11 +120,117 @@ func NewServer() *Server {
s.logger = log.New(s.LogOutput, "", log.LstdFlags)
s.Handler.Holder = s.Holder
s.diagnostics.server = s
return s
}
// Open opens and initializes the server.
func (s *Server) Open() error {
s.Logger().Printf("open server")
// s.ln can be configured prior to Open() via s.OpenListener().
if s.ln == nil {
if err := s.OpenListener(); err != nil {
return err
}
}
// Get or create NodeID.
s.NodeID = s.LoadNodeID()
// Set Cluster Node.
node := &Node{
ID: s.NodeID,
URI: s.URI,
IsCoordinator: s.Cluster.Coordinator == s.NodeID,
}
s.Cluster.Node = node
// Append the NodeID tag to stats.
s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID))
// Peek at the holder to determine if there is data on disk.
// Don't actually load the data until after the Cluster
// management starts.
s.Holder.LogOutput = s.LogOutput
s.Holder.Peek()
// Create default HTTP client
s.createDefaultClient(s.RemoteClient)
// Create executor for executing queries.
e := NewExecutor(s.RemoteClient)
e.Holder = s.Holder
e.Node = node
e.Cluster = s.Cluster
e.MaxWritesPerRequest = s.MaxWritesPerRequest
// Cluster settings.
s.Cluster.Broadcaster = s.Broadcaster
s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
// Initialize HTTP handler.
s.Handler.Broadcaster = s.Broadcaster
s.Handler.BroadcastHandler = s
s.Handler.StatusHandler = s
s.Handler.Node = node
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Handler.LogOutput = s.LogOutput
s.Cluster.prefect = s.Handler
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
err := http.Serve(s.ln, s.Handler)
if err != nil {
s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
}
// Open Cluster management.
if err := s.Cluster.Open(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
}
// Open holder.
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.Cluster.SetNodeState(NodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
// Listen for joining nodes.
// This needs to start after the Holder has opened so that nodes can join
// the cluster without waiting for data to load on the coordinator. Before
// this starts, the joins are queued up in the Cluster.joiningLeavingNodes
// buffered channel.
s.Cluster.ListenForJoins()
// Start background monitoring.
s.wg.Add(3)
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
return nil
}
// OpenListener opens a listener for the Server.
func (s *Server) OpenListener() error {
s.Logger().Printf("open server listener: %s", s.URI)
if s.ln != nil {
return fmt.Errorf("a listener already exists for server: %s", s.URI)
}
var ln net.Listener
var err error
@ -148,87 +258,6 @@ func (s *Server) Open() error {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
// Create local node if no cluster is specified.
if len(s.Cluster.Nodes) == 0 {
s.Cluster.Nodes = []*Node{
{Scheme: s.URI.Scheme(), Host: s.URI.HostPort()},
}
}
for i, n := range s.Cluster.Nodes {
if s.Cluster.NodeByHost(n.Host) != nil {
s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i))
}
}
// Open holder.
s.Holder.LogOutput = s.LogOutput
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
}
// Open NodeSet communication
if err := s.Cluster.NodeSet.Open(); err != nil {
return fmt.Errorf("opening NodeSet: %v", err)
}
// Create default HTTP client
s.createDefaultClient(s.RemoteClient)
// Create executor for executing queries.
e := NewExecutor(s.RemoteClient)
e.Holder = s.Holder
e.Scheme = s.URI.Scheme()
e.Host = s.URI.HostPort()
e.Cluster = s.Cluster
e.MaxWritesPerRequest = s.MaxWritesPerRequest
s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
// Initialize HTTP handler.
s.Handler.Broadcaster = s.Broadcaster
s.Handler.BroadcastHandler = s
s.Handler.StatusHandler = s
s.Handler.URI = s.URI
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Handler.LogOutput = s.LogOutput
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
server := &http.Server{Handler: s.Handler}
go func() {
<-s.closing
server.Close()
}()
err := server.Serve(ln)
if err != nil && err.Error() != "http: Server closed" {
s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// load local ID
if err := s.Holder.loadLocalID(); err != nil {
s.Logger().Println(err)
}
if err := s.loadClusterID(); err != nil {
s.Logger().Println(err)
}
// Start background monitoring.
s.wg.Add(4)
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorMaxSlices() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
return nil
}
@ -241,6 +270,9 @@ func (s *Server) Close() error {
if s.ln != nil {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.Close()
}
if s.Holder != nil {
s.Holder.Close()
}
@ -248,6 +280,20 @@ func (s *Server) Close() error {
return nil
}
// LoadNodeID gets NodeID from disk, or creates a new value.
// If server.NodeID is already set, a new ID is not created.
func (s *Server) LoadNodeID() string {
if s.NodeID != "" {
return s.NodeID
}
nodeID, err := s.Holder.loadNodeID()
if err != nil {
s.Logger().Printf("loading NodeID: %v", err)
return s.NodeID
}
return nodeID
}
// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
if s.ln == nil {
@ -298,7 +344,7 @@ func (s *Server) monitorAntiEntropy() {
// Initialize syncer with local holder and remote client.
var syncer HolderSyncer
syncer.Holder = s.Holder
syncer.URI = s.URI
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.RemoteClient
@ -317,44 +363,6 @@ func (s *Server) monitorAntiEntropy() {
}
}
// monitorMaxSlices periodically pulls the highest slice from each node in the cluster.
func (s *Server) monitorMaxSlices() {
// Ignore if only one node in the cluster.
if len(s.Cluster.Nodes) <= 1 {
return
}
ticker := time.NewTicker(s.PollingInterval)
defer ticker.Stop()
for {
select {
case <-s.closing:
return
case <-ticker.C:
}
oldmaxslices := s.Holder.MaxSlices()
for _, node := range s.Cluster.Nodes {
if s.URI.HostPort() != node.Host {
maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host)
for index, newmax := range maxSlices {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex := s.Holder.Index(index); localIndex != nil {
if newmax > oldmaxslices[index] {
oldmaxslices[index] = newmax
localIndex.SetRemoteMaxSlice(newmax)
}
} else {
s.Logger().Printf("Local Index not found: %s", index)
}
}
}
}
}
}
// ReceiveMessage represents an implementation of BroadcastHandler.
func (s *Server) ReceiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
@ -386,16 +394,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := FrameOptions{
RowLabel: obj.Meta.RowLabel,
InverseEnabled: obj.Meta.InverseEnabled,
RangeEnabled: obj.Meta.RangeEnabled,
CacheType: obj.Meta.CacheType,
CacheSize: obj.Meta.CacheSize,
TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum),
Fields: decodeFields(obj.Meta.Fields),
}
_, err := idx.CreateFrame(obj.Frame, opt)
opt := decodeFrameOptions(obj.Meta)
_, err := idx.CreateFrame(obj.Frame, *opt)
if err != nil {
return err
}
@ -404,6 +404,17 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err := idx.DeleteFrame(obj.Frame); err != nil {
return err
}
case *internal.CreateFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
field := decodeField(obj.Field)
if err := f.CreateField(field); err != nil {
return err
}
case *internal.DeleteFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if err := f.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateInputDefinitionMessage:
idx := s.Holder.Index(obj.Index)
if idx == nil {
@ -416,6 +427,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.CreateViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
}
_, _, err := f.createViewIfNotExistsBase(obj.View)
if err != nil {
return err
}
case *internal.DeleteViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {
@ -425,7 +445,36 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.ClusterStatus:
err := s.Cluster.MergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
err := s.Cluster.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
err := s.Cluster.MarkResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.Cluster.SetCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.Cluster.UpdateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
case *internal.RecalculateCaches:
s.Holder.RecalculateCaches()
case *internal.NodeEventMessage:
s.Cluster.ReceiveEvent(DecodeNodeEvent(obj))
}
return nil
}
@ -433,17 +482,13 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
func (s *Server) SendSync(pb proto.Message) error {
var eg errgroup.Group
for _, node := range s.Cluster.Nodes {
uri, err := node.URI()
if err != nil {
return err
}
s.Logger().Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if *s.URI == *uri {
if s.URI == node.URI {
continue
}
ctx := context.WithValue(context.Background(), "uri", uri)
ctx := context.WithValue(context.Background(), "uri", &node.URI)
eg.Go(func() error {
return s.defaultClient.SendMessage(ctx, pb)
})
@ -457,150 +502,145 @@ func (s *Server) SendAsync(pb proto.Message) error {
return s.Gossiper.SendAsync(pb)
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
s.Logger().Printf("SendTo: %s", to.URI)
ctx := context.WithValue(context.Background(), "uri", &to.URI)
return s.defaultClient.SendMessage(ctx, pb)
}
// Server implements StatusHandler.
// LocalStatus is used to periodically sync information
// between nodes. Under normal conditions, nodes should
// remain in sync through Broadcast messages. For cases
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice/MaxInverseSlice by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Cluster == nil {
return nil, errors.New("Server.Cluster is nil")
}
if s.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Scheme: s.URI.Scheme(),
Host: s.URI.HostPort(),
State: NodeStateUp,
Indexes: EncodeIndexes(s.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI.HostPort())
Node: EncodeNode(s.Cluster.Node),
MaxSlices: s.Holder.EncodeMaxSlices(),
Schema: s.Holder.EncodeSchema(),
}
return &ns, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
// ClusterStatus returns the ClusterState and NodeSet for the cluster.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Update local Node.state.
ns, err := s.LocalStatus()
if err != nil {
return nil, err
}
node := s.Cluster.NodeByHost(s.URI.HostPort())
node.SetStatus(ns.(*internal.NodeStatus))
// Update NodeState for all nodes.
for host, nodeState := range s.Cluster.NodeStates() {
// In a default configuration (or single-node) where a StaticNodeSet is used
// then all nodes are marked as DOWN. At the very least, we should consider
// the local node as UP.
// TODO: we should be able to remove this check if/when cluster.Nodes and
// cluster.NodeSet are unified.
if host == s.URI.HostPort() {
nodeState = NodeStateUp
}
node := s.Cluster.NodeByHost(host)
node.SetState(nodeState)
}
return s.Cluster.Status(), nil
}
// HandleRemoteStatus receives incoming NodeState from remote nodes.
// HandleRemoteStatus receives incoming NodeStatus from remote nodes.
func (s *Server) HandleRemoteStatus(pb proto.Message) error {
return s.mergeRemoteStatus(pb.(*internal.NodeStatus))
// Ignore NodeStatus messages until the cluster is in a Normal state.
if s.Cluster.State() != ClusterStateNormal {
return nil
}
go func() {
// Make sure the holder has opened.
<-s.Holder.opened
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
if err != nil {
s.Logger().Printf("merge remote status: %s", err)
}
}()
return nil
}
func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// Update Node.state.
node := s.Cluster.NodeByHost(ns.Host)
node.SetStatus(ns)
// Ignore status updates from self.
if s.NodeID == DecodeNode(ns.Node).ID {
return nil
}
// Create indexes that don't exist.
for _, index := range ns.Indexes {
opt := IndexOptions{
ColumnLabel: index.Meta.ColumnLabel,
TimeQuantum: TimeQuantum(index.Meta.TimeQuantum),
// Sync schema.
if err := s.Holder.ApplySchema(ns.Schema); err != nil {
return err
}
// Sync maxSlices (standard).
oldmaxslices := s.Holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
continue
}
idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return err
if newMax > oldmaxslices[index] {
oldmaxslices[index] = newMax
localIndex.SetRemoteMaxSlice(newMax)
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := FrameOptions{
RowLabel: f.Meta.RowLabel,
TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),
CacheSize: f.Meta.CacheSize,
}
_, err := idx.CreateFrameIfNotExists(f.Name, opt)
if err != nil {
return err
}
}
// Sync maxSlices (inverse).
oldMaxInverseSlices := s.Holder.MaxInverseSlices()
for index, newMaxInverse := range ns.MaxSlices.Inverse {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
oldMaxInverseSlices[index] = newMaxInverse
localIndex.SetRemoteMaxInverseSlice(newMaxInverse)
}
}
return nil
}
func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) {
// Create HTTP request.
req, err := http.NewRequest("GET", (&url.URL{
Scheme: scheme,
Host: hostPort,
Path: "/slices/max",
}).String(), nil)
if err != nil {
return nil, err
}
// Require protobuf encoding.
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("User-Agent", "pilosa/"+Version)
nodeURI, err := NewURIFromAddress(hostPort)
if err != nil {
return nil, err
}
nodeURI.SetScheme(scheme)
ctx := context.WithValue(context.Background(), "uri", nodeURI)
return s.defaultClient.MaxSliceByIndex(ctx)
}
// monitorDiagnostics periodically polls the Pilosa Indexes for cluster info.
func (s *Server) monitorDiagnostics() {
if s.DiagnosticInterval <= 0 {
// Do not send more than once a minute
if s.DiagnosticInterval < time.Minute {
s.Logger().Printf("diagnostics disabled")
return
} else {
s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
}
s.diagnostics.SetLogger(s.LogOutput)
s.diagnostics.SetVersion(Version)
s.diagnostics.SetInterval(s.DiagnosticInterval)
s.diagnostics.Open()
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","))
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("LocalID", s.Holder.LocalID)
s.diagnostics.Set("ClusterID", s.ClusterID)
s.diagnostics.Set("NodeID", s.NodeID)
s.diagnostics.Set("ClusterID", s.Cluster.ID)
s.diagnostics.EnrichWithOSInfo()
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder)
openFiles, err := CountOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
s.diagnostics.Set("GoRoutines", runtime.NumGoroutine())
s.diagnostics.EnrichWithMemoryInfo()
s.diagnostics.EnrichWithSchemaProperties()
s.diagnostics.CheckVersion()
s.diagnostics.Flush()
err = s.diagnostics.Flush()
if err != nil {
s.Logger().Printf("Diagnostics error: %s", err)
}
}
ticker := time.NewTicker(s.DiagnosticInterval)
@ -628,8 +668,7 @@ func (s *Server) monitorRuntime() {
ticker := time.NewTicker(s.MetricInterval)
defer ticker.Stop()
gcn := gcnotifier.New()
defer gcn.Close()
defer s.GCNotifier.Close()
s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
@ -638,7 +677,7 @@ func (s *Server) monitorRuntime() {
select {
case <-s.closing:
return
case <-gcn.AfterGC():
case <-s.GCNotifier.AfterGC():
// GC just ran.
s.Holder.Stats.Count("garbage_collection", 1, 1.0)
case <-ticker.C:
@ -667,25 +706,6 @@ func (s *Server) createDefaultClient(remoteClient *http.Client) {
s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient)
}
func (s *Server) loadClusterID() error {
// If this is the first node in the cluster, set the ClusterID to its ID
node0URI, err := s.Cluster.Nodes[0].URI()
if err == nil {
if s.URI.Equals(node0URI) {
s.ClusterID = s.Holder.LocalID
return nil
}
} else {
return err
}
if clusterID, err := s.defaultClient.NodeID(node0URI); err == nil {
s.ClusterID = clusterID
return nil
} else {
return err
}
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
switch runtime.GOOS {
@ -707,47 +727,11 @@ func CountOpenFiles() (int, error) {
}
}
// StatusHandler specifies two methods which an object must implement to share
// state in the cluster. These are used by the GossipNodeSet to implement the
// StatusHandler specifies the methods which an object must implement to share
// state in the cluster. These are used by the GossipMemberSet to implement the
// LocalState and MergeRemoteState methods of memberlist.Delegate
type StatusHandler interface {
LocalStatus() (proto.Message, error)
ClusterStatus() (proto.Message, error)
HandleRemoteStatus(proto.Message) error
}
type diagnosticsFrameProperties struct {
BSIFieldCount int
TimeQuantumEnabled bool
}
func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) {
// NOTE: this function is not in the diagnostics package, since circular imports are not allowed.
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields.Fields)
}
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}

581
server/cluster_test.go Normal file
View file

@ -0,0 +1,581 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server_test
import (
"context"
"fmt"
"net/http"
"reflect"
"strings"
"testing"
"time"
"golang.org/x/sync/errgroup"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/test"
)
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
m0 := test.MustRunMain()
defer m0.Close()
m1 := test.MustRunMain()
defer m1.Close()
// Update cluster config
m0.Server.Cluster.Nodes = []*pilosa.Node{
{ID: m0.Server.NodeID, URI: m0.Server.URI},
{ID: m1.Server.NodeID, URI: m1.Server.URI},
}
m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
// Configure node0
// get the host portion of addr to use for binding
m0.Config.Gossip.Port = "0"
m0.Config.Gossip.Seeds = []string{}
m0.Server.Cluster.Coordinator = m0.Server.NodeID
m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}}
m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput)
gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server)
if err != nil {
t.Fatal(err)
}
m0.Server.Cluster.MemberSet = gossipMemberSet0
m0.Server.Broadcaster = m0.Server
m0.Server.Gossiper = gossipMemberSet0
m0.Server.Handler.Broadcaster = m0.Server.Broadcaster
m0.Server.Holder.Broadcaster = m0.Server.Broadcaster
m0.Server.BroadcastReceiver = gossipMemberSet0
if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m0.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
// Configure node1
// get the host portion of addr to use for binding
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()}
m1.Server.Cluster.Coordinator = m0.Server.NodeID
m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput)
gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server)
if err != nil {
t.Fatal(err)
}
m1.Server.Cluster.MemberSet = gossipMemberSet1
m1.Server.Broadcaster = m1.Server
m1.Server.Gossiper = gossipMemberSet1
m1.Server.Handler.Broadcaster = m1.Server.Broadcaster
m1.Server.Holder.Broadcaster = m1.Server.Broadcaster
m1.Server.BroadcastReceiver = gossipMemberSet1
if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m1.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Expected indexes and Frames
expected := map[string][]string{
"i": []string{"f"},
}
// Create a client for each node.
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Make sure node0 knows about the index and frame created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
}
// Make sure node1 knows about the index and frame created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
}
}
if !reflect.DeepEqual(received1, expected) {
t.Fatalf("unexpected schema on node1: %s", received1)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
// Make sure node0 knows about the latest MaxSlice.
maxSlices0, err := client0.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices0["i"] != 2 {
t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"])
}
// Make sure node1 knows about the latest MaxSlice.
maxSlices1, err := client1.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
}
// Write input definition to the first node.
if _, err := m0.CreateDefinition("i", "test", `{
"frames": [{"name": "event-time",
"options": {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "columnID",
"primaryKey": true
}]}
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
frame0 := m0.Server.Holder.Frame("i", "event-time")
if frame0 == nil {
t.Fatal("frame not found")
}
frame1 := m1.Server.Holder.Frame("i", "event-time")
if frame1 == nil {
t.Fatal("frame not found")
}
}
// Ensure that an empty node comes up in a NORMAL state.
func TestClusterResize_EmptyNode(t *testing.T) {
m0 := test.MustRunMain()
defer m0.Close()
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State())
}
}
// Ensure that a cluster of empty nodes comes up in a NORMAL state.
func TestClusterResize_EmptyNodes(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
gossipHost := "localhost"
gossipPort := 0
seed, err := m0.RunWithTransport(gossipHost, gossipPort, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
seed, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed})
if err != nil {
t.Fatal(err)
}
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
}
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNode(t *testing.T) {
t.Run("NoData", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("WithIndex", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("ContinuousSlices", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=1300000)
`); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("SkippedSlice", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
}
// Ensure that redundant gossip seeds are used
func TestCluster_GossipMembership(t *testing.T) {
t.Run("Node0Down", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
// Pass invalid seed as first in list
_, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed})
if err != nil {
return err
}
return nil
})
// Configure node2
m2 := test.NewMainWithCluster(false)
defer m2.Close()
eg.Go(func() error {
// Pass invalid seed as last in list
_, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
} else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State())
}
numNodes := len(m0.Server.Cluster.Status().Nodes)
if numNodes != 3 {
t.Fatalf("Expected 3 nodes, got %d", numNodes)
}
})
}
func TestClusterResize_RemoveNode(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3)
m0 := cluster[0]
m1 := cluster[1]
t.Run("ErrorRemoveInvalidNode", func(t *testing.T) {
resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`)
expBody := "Node is not a member of the cluster: invalid-node-id"
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveCoordinator", func(t *testing.T) {
resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator."
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveOnNonCoordinator", func(t *testing.T) {
resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "")
coordinatorNodeID := resp.Body
resp = test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID)
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) {
client0 := m0.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// This is an attempt to ensure there is data on both nodes, but is not guaranteed.
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {
t.Fatal(err)
}
resp := test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := "not enough data to perform resize"
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if !strings.Contains(resp.Body, expBody) {
t.Fatalf("expected to contain '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
}

165
server/server.go
View file

@ -32,10 +32,12 @@ import (
"crypto/tls"
"io/ioutil"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/statsd"
)
@ -59,10 +61,13 @@ type Command struct {
CPUProfile string
CPUTime time.Duration
// Gossip transport
GossipTransport *gossip.Transport
// Standard input/output
*pilosa.CmdIO
// running will be closed once Command.Run is finished.
// Started will be closed once Command.Run is finished.
Started chan struct{}
// Done will be closed when Command.Close() is called
Done chan struct{}
@ -99,12 +104,18 @@ func (m *Command) Run(args ...string) (err error) {
return err
}
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
return err
}
// Initialize server.
if err = m.Server.Open(); err != nil {
return fmt.Errorf("server.Open: %v", err)
}
m.Server.Logger().Printf("Listening as %s\n", m.Server.URI.Normalize())
m.Server.Logger().Printf("Listening as %s\n", m.Server.URI)
return nil
}
@ -116,24 +127,16 @@ func (m *Command) SetupServer() error {
}
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
return err
}
m.Server.URI = uri
m.Server.URI = *uri
cluster := pilosa.NewCluster()
cluster.ReplicaN = m.Config.Cluster.ReplicaN
cluster.Holder = m.Server.Holder
for _, address := range m.Config.Cluster.Hosts {
uri, err := pilosa.NewURIFromAddress(address)
if err != nil {
return err
}
cluster.Nodes = append(cluster.Nodes, &pilosa.Node{
Scheme: uri.Scheme(),
Host: uri.HostPort(),
})
}
m.Server.Cluster = cluster
// Setup logging output.
@ -142,6 +145,12 @@ func (m *Command) SetupServer() error {
return err
}
// Configure data directory (for Cluster .topology)
m.Server.Cluster.Path = m.Config.DataDir
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = boltdb.NewAttrStore
// Configure holder.
m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Holder.Path = m.Config.DataDir
@ -149,6 +158,8 @@ func (m *Command) SetupServer() error {
if m.Config.Metric.Diagnostics {
m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval)
}
m.Server.SystemInfo = gopsutil.NewSystemInfo()
m.Server.GCNotifier = gcnotify.NewActiveGCNotifier()
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
if err != nil {
return err
@ -177,68 +188,15 @@ func (m *Command) SetupServer() error {
InsecureSkipVerify: m.Config.TLS.SkipVerify,
}
// TODO Review this location
TLSConfig = m.Server.TLS
}
c := pilosa.GetHTTPClient(TLSConfig)
m.Server.RemoteClient = c
m.Server.Handler.RemoteClient = c
m.Server.Cluster.RemoteClient = c
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
// Config.GossipPort is deprecated, so Config.Gossip.Port has priority
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
} else if m.Config.GossipPort != "" {
gossipPortStr = m.Config.GossipPort
}
switch m.Config.Cluster.Type {
case pilosa.ClusterGossip:
gossipPort, err := strconv.Atoi(gossipPortStr)
if err != nil {
return err
}
gossipSeed := pilosa.DefaultHost + ":" + pilosa.DefaultGossipPort
// Config.GossipSeed is deprecated, so Config.Gossip.Seed has priority
if m.Config.Gossip.Seed != "" {
gossipSeed = m.Config.Gossip.Seed
} else if m.Config.GossipSeed != "" {
gossipSeed = m.Config.GossipSeed
}
var gossipKey []byte
if m.Config.Gossip.Key != "" {
gossipKey, err = ioutil.ReadFile(m.Config.Gossip.Key)
if err != nil {
return err
}
}
// get the host portion of addr to use for binding
gossipHost := uri.Host()
gossipNodeSet, err := gossip.NewGossipNodeSet(uri.HostPort(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey)
if err != nil {
return err
}
m.Server.Cluster.NodeSet = gossipNodeSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipNodeSet
m.Server.Gossiper = gossipNodeSet
case pilosa.ClusterStatic, pilosa.ClusterNone:
m.Server.Broadcaster = pilosa.NopBroadcaster
m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
m.Server.Gossiper = pilosa.NopGossiper
err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
if err != nil {
return err
}
default:
return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type)
}
// Statik file system.
m.Server.Handler.FileSystem = &statik.FileSystem{}
// Set configuration options.
m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
@ -246,6 +204,71 @@ func (m *Command) SetupServer() error {
return nil
}
// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {
m.Server.NodeID = m.Server.LoadNodeID()
if m.Config.Cluster.Disabled {
m.Server.Cluster.Static = true
m.Server.Cluster.Coordinator = m.Server.NodeID
for _, address := range m.Config.Cluster.Hosts {
uri, err := pilosa.NewURIFromAddress(address)
if err != nil {
return err
}
m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{
URI: *uri,
})
}
m.Server.Broadcaster = pilosa.NopBroadcaster
m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes)
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
m.Server.Gossiper = pilosa.NopGossiper
return nil
}
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
}
gossipPort, err := strconv.Atoi(gossipPortStr)
if err != nil {
return err
}
// get the host portion of addr to use for binding
gossipHost := m.Server.URI.Host()
var transport *gossip.Transport
if m.GossipTransport != nil {
transport = m.GossipTransport
} else {
transport, err = gossip.NewTransport(gossipHost, gossipPort)
if err != nil {
return err
}
}
// Set Coordinator.
if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 {
m.Server.Cluster.Coordinator = m.Server.NodeID
}
m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput)
gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server)
if err != nil {
return err
}
m.Server.Cluster.MemberSet = gossipMemberSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipMemberSet
m.Server.Gossiper = gossipMemberSet
return nil
}
// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path.
func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) {
// This is split out so it can be used in NewServeCmd as well as SetupServer

323
server/server_test.go
View file

@ -15,26 +15,20 @@
package server_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
"reflect"
"runtime"
"sort"
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -45,7 +39,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
if err := quick.Check(func(cmds []SetCommand) bool {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create client.
@ -121,7 +115,7 @@ func TestMain_Set_Quick(t *testing.T) {
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -198,7 +192,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -242,7 +236,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Ensure program can set column attributes with columnLabel option.
func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -276,11 +270,12 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := NewMainArrayWithCluster(2)
m0 := mains1[0]
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
m11 := mains1[1]
// Create frames.
client := m0.Client()
client := m10.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
@ -289,7 +284,7 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Write data on first cluster.
if _, err := m0.Query("i", "", `
if _, err := m10.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=100)
SetBit(rowID=1, frame="f", columnID=1000)
SetBit(rowID=1, frame="f", columnID=100000)
@ -302,34 +297,45 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Query row on first cluster.
if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Start second cluster.
mains2 := NewMainArrayWithCluster(2)
m2 := mains2[0]
defer m2.Close()
mains2 := test.MustRunMainWithCluster(t, 2)
m20 := mains2[0]
defer m20.Close()
m21 := mains2[1]
defer m21.Close()
// Import from first cluster.
client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create new index:", err)
}
if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create new frame:", err)
}
if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil {
if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
// Query row on second cluster.
if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
@ -378,226 +384,45 @@ func TestCountOpenFiles(t *testing.T) {
}
}
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
mains := NewMainArrayWithCluster(2)
m0 := mains[0]
defer m0.Close()
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)
m1 := mains[1]
defer m1.Close()
// Expected indexes and Frames
expected := map[string][]string{
"i": []string{"f"},
}
// Create a client for each node.
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
// Create the schema.
client0 := cluster[0].Client()
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
t.Fatal("create index:", err)
}
if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create frame:", err)
}
// Make sure node0 knows about the index and frame created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
// Set some bits
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID))
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
t.Fatal("setting bits:", err)
}
// Make sure node1 knows about the index and frame created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
// Calculate caches on the first node
cluster[0].RecalculateCaches()
target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}`
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(frame="f")`)
if err != nil {
t.Fatal(err)
}
res = strings.TrimSpace(res)
if sortedString(target) != sortedString(res) {
t.Fatalf("%v != %v", target, res)
}
}
if !reflect.DeepEqual(received1, expected) {
t.Fatalf("unexpected schema on node1: %s", received1)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
// Make sure node0 knows about the latest MaxSlice.
maxSlices0, err := client0.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices0["i"] != 2 {
t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"])
}
// Make sure node1 knows about the latest MaxSlice.
maxSlices1, err := client1.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
}
// Write input definition to the first node.
if _, err := m0.CreateDefinition("i", "test", `{
"frames": [{"name": "event-time",
"options": {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "columnID",
"primaryKey": true
}]}
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
frame0 := m0.Server.Holder.Frame("i", "event-time")
if frame0 == nil {
t.Fatal("frame not found")
}
frame1 := m1.Server.Holder.Frame("i", "event-time")
if frame1 == nil {
t.Fatal("frame not found")
}
}
// Main represents a test wrapper for main.Main.
type Main struct {
*server.Command
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *test.Network
m.Config.DataDir = path
m.Config.Bind = "localhost:0"
m.Config.Cluster.Type = "static"
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
}
return m
}
func NewMainArrayWithCluster(size int) []*Main {
cluster, err := test.NewServerCluster(size)
if err != nil {
panic(err)
}
mainArray := make([]*Main, size)
for i := 0; i < size; i++ {
mainArray[i] = &Main{Command: cluster.Servers[i]}
}
return mainArray
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
if err := m.Run(); err != nil {
panic(err)
}
return m
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
// Create new main with the same config.
config := m.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
m.Server.Network = *test.Network
m.Config = config
// Run new program.
if err := m.Run(); err != nil {
return err
}
return nil
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
panic(err)
}
return client
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// CreateDefinition.
func (m *Main) CreateDefinition(index, def, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// SetCommand represents a command to set a bit.
@ -656,32 +481,6 @@ func ParseConfig(s string) (pilosa.Config, error) {
return c, err
}
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.
func MustDo(method, urlStr string, body string) *httpResponse {
req, err := http.NewRequest(method, urlStr, strings.NewReader(body))
if err != nil {
panic(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
return &httpResponse{Response: resp, Body: string(buf)}
}
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*http.Response
Body string
}
// MustMarshalJSON marshals v into a string. Panic on error.
func MustMarshalJSON(v interface{}) string {
buf, err := json.Marshal(v)
@ -691,6 +490,12 @@ func MustMarshalJSON(v interface{}) string {
return string(buf)
}
func sortedString(s string) string {
arr := strings.Split(s, "")
sort.Strings(arr)
return strings.Join(arr, "")
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

2
statik/.gitignore vendored
View file

@ -1 +1 @@
statik.go
/statik.go

3
statik/doc.go
View file

@ -1,3 +0,0 @@
// Package statik contains static assets for the Web UI. `go generate` will
// produce statik.go, which is ignored by git.
package statik

37
statik/filesystem.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//go:generate statik -src=../webui -dest=..
//
// Package statik contains static assets for the Web UI. `go generate` or
// `make generate-statik` will produce statik.go, which is ignored by git.
package statik
import (
"net/http"
"github.com/pilosa/pilosa"
"github.com/rakyll/statik/fs"
)
// Ensure nopFileSystem implements interface.
var _ pilosa.FileSystem = &FileSystem{}
// FileSystem represents a static FileSystem.
type FileSystem struct{}
// New is a statik implementation of FileSystem New method.
func (s *FileSystem) New() (http.FileSystem, error) {
return fs.New()
}

2
statsd/statsd.go
View file

@ -24,7 +24,7 @@ import (
"github.com/pilosa/pilosa"
)
// StatsD protocal wrapper using the DataDog library that added Tags to the StatsD protocal
// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol
// statsD defailt host is "127.0.0.1:8125"
const (

11
test/attr.go
View file

@ -22,15 +22,16 @@ import (
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
*pilosa.AttrStore
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore() *AttrStore {
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
@ -38,7 +39,7 @@ func NewAttrStore() *AttrStore {
f.Close()
os.Remove(f.Name())
return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
@ -74,8 +75,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) {
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() *AttrStore {
s := NewAttrStore()
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}

398
test/cluster.go
View file

@ -15,24 +15,42 @@
package test
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
path, err := ioutil.TempDir("", "pilosa-cluster-")
if err != nil {
panic(err)
}
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Scheme: "http",
Host: fmt.Sprintf("host%d", i),
ID: fmt.Sprintf("node%d", i),
URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
c.Node = c.Nodes[0]
c.Coordinator = c.Nodes[0].ID
return c
}
@ -53,3 +71,379 @@ type ConstHasher struct {
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// NewURI is a test URI creator that intentionally swallows errors.
func NewURI(scheme, host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewURIFromHostPort(host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// TestCluster represents a cluster of test nodes, each of which
// has a pilosa.Cluster.
type TestCluster struct {
Clusters []*pilosa.Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*pilosa.Node
}
func (t *TestCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *TestCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &pilosa.NodeEvent{
Event: pilosa.NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != pilosa.ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &pilosa.Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := pilosa.NewHolder()
h.Path = path
// cluster
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
c.Holder = h
c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the TestCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewTestCluster returns a new instance of test.Cluster.
func NewTestCluster(n int) *TestCluster {
tc := &TestCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *TestCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *TestCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(pilosa.NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *TestCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// TestCluster implements Broadcaster interface.
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *TestCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == pilosa.ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *TestCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := pilosa.DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

5
test/executor.go
View file

@ -34,14 +34,13 @@ func init() {
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
// The executor always matches the uri of the first cluster node.
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
executor := pilosa.NewExecutor(remoteClient)
e := &Executor{Executor: executor}
e.Holder = holder
e.Cluster = cluster
e.Scheme = cluster.Nodes[0].Scheme
e.Host = cluster.Nodes[0].Host
e.Node = cluster.Nodes[0]
return e
}

6
test/fragment.go
View file

@ -27,7 +27,7 @@ const SliceWidth = pilosa.SliceWidth
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
RowAttrStore *AttrStore
RowAttrStore pilosa.AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
@ -43,7 +43,7 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
return f
}
@ -78,7 +78,7 @@ func (f *Fragment) Reopen() error {
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
return err
}

48
test/handler.go
View file

@ -17,7 +17,6 @@ package test
import (
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
@ -47,6 +46,8 @@ func NewHandler() *Handler {
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
h.SetNormal()
return h
}
@ -80,43 +81,36 @@ func NewServer() *Server {
if err != nil {
panic(err)
}
s.Handler.URI = uri
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Node = s.Handler.Cluster.Nodes[0]
return s
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
// LocalStatus exists so that test.Server implements StatusHandler.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Handler.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Host: s.Handler.Handler.URI.HostPort(),
State: pilosa.NodeStateUp,
Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.URI.HostPort())
}
return &ns, nil
return nil, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
// ClusterStatus exists so that test.Server implements StatusHandler.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Assuming we are only testing this with one Node
// So just return its status
return s.LocalStatus()
id := "test-node"
uri := pilosa.DefaultURI()
node := &pilosa.Node{
ID: id,
URI: *uri,
}
return &internal.ClusterStatus{
ClusterID: "",
State: pilosa.ClusterStateNormal,
Nodes: pilosa.EncodeNodes([]*pilosa.Node{node}),
}, nil
}
// HandleRemoteStatus just need to implement a nop to complete the Interface
@ -125,12 +119,12 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil }
// Host returns the hostname of the running server.
func (s *Server) Host() string { return MustParseURLHost(s.URL) }
func (s *Server) HostURI() *pilosa.URI {
func (s *Server) HostURI() pilosa.URI {
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
panic(err)
}
return uri
return *uri
}
// MustParseURLHost parses rawurl and returns the hostname. Panic on error.

3
test/holder.go
View file

@ -20,6 +20,7 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// Holder is a test wrapper for pilosa.Holder.
@ -38,6 +39,7 @@ func NewHolder() *Holder {
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h
}
@ -64,6 +66,7 @@ func (h *Holder) Reopen() error {
h.Holder = pilosa.NewHolder()
h.Holder.Path = path
h.Holder.LogOutput = logOutput
h.Holder.NewAttrStore = boltdb.NewAttrStore
if err := h.Holder.Open(); err != nil {
return err
}

317
test/pilosa.go
View file

@ -16,119 +16,268 @@ package test
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net"
"strconv"
"net/http"
"os"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
func MustNewRunningServer(t *testing.T) *server.Command {
s, err := newServer()
if err != nil {
t.Fatalf("getting new server: %v", err)
}
////////////////////////////////////////////////////////////////////////////////////
// Main represents a test wrapper for main.Main.
type Main struct {
*server.Command
err = s.Run()
if err != nil {
t.Fatalf("running new pilosa server: %v", err)
}
return s
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
func newServer() (*server.Command, error) {
s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
port, err := findPort()
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
return nil, errors.Wrap(err, "getting port")
panic(err)
}
s.Config.Bind = "localhost:" + strconv.Itoa(port)
gport, err := findPort()
if err != nil {
return nil, errors.Wrap(err, "getting gossip port")
}
s.Config.GossipPort = strconv.Itoa(gport)
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *Network
m.Server.NewAttrStore = NewAttrStore
m.Server.Holder.NewAttrStore = NewAttrStore
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
s.Config.GossipSeed = "localhost:" + s.Config.GossipPort
s.Config.Cluster.Type = "gossip"
td, err := ioutil.TempDir("", "")
if err != nil {
return nil, errors.Wrap(err, "temp dir")
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
}
s.Config.DataDir = td
return s, nil
return m
}
func findPort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", ":0")
if err != nil {
return 0, errors.Wrap(err, "resolving new port addr")
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, errors.Wrap(err, "listening to get new port")
}
port := l.Addr().(*net.TCPAddr).Port
err = l.Close()
if err != nil {
return port, errors.Wrap(err, "closing listener")
}
return port, nil
// NewMainWithCluster returns a new instance of Main with clustering enabled.
func NewMainWithCluster(isCoordinator bool) *Main {
m := NewMain()
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
}
func MustFindPort(t *testing.T) int {
port, err := findPort()
// MustRunMainWithCluster ruturns a running array of *Main where
// all nodes are joined via memberlist (i.e. clustering enabled).
func MustRunMainWithCluster(t *testing.T, size int) []*Main {
ma, err := runMainWithCluster(size)
if err != nil {
t.Fatalf("allocating new port: %v", err)
t.Fatalf("new main array with cluster: %v", err)
}
return port
return ma
}
type Cluster struct {
Servers []*server.Command
}
func MustNewServerCluster(t *testing.T, size int) *Cluster {
cluster, err := NewServerCluster(size)
if err != nil {
t.Fatalf("new cluster: %v", err)
// runMainWithCluster runs an array of *Main where all nodes are
// joined via memberlist (i.e. clustering enabled).
func runMainWithCluster(size int) ([]*Main, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
return cluster
}
func NewServerCluster(size int) (cluster *Cluster, err error) {
cluster = &Cluster{
Servers: make([]*server.Command, size),
}
hosts := make([]string, size)
mains := make([]*Main, size)
gossipHost := "localhost"
gossipPort := 0
var err error
var gossipSeeds = make([]string, size)
for i := 0; i < size; i++ {
s, err := newServer()
m := NewMainWithCluster(i == 0)
gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i])
if err != nil {
return nil, errors.Wrap(err, "new server")
return nil, errors.Wrap(err, "RunWithTransport")
}
cluster.Servers[i] = s
hosts[i] = s.Config.Bind
s.Config.GossipSeed = cluster.Servers[0].Config.GossipSeed
mains[i] = m
}
for _, s := range cluster.Servers {
s.Config.Cluster.Hosts = hosts
}
for i, s := range cluster.Servers {
err := s.Run()
if err != nil {
for j := 0; j <= i; j++ {
cluster.Servers[j].Close()
}
return nil, errors.Wrapf(err, "starting server %d of %d. Config: %#v", i+1, size, s.Config)
}
}
return cluster, nil
return mains, nil
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Run(); err != nil {
panic(err)
}
return m
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
// Create new main with the same config.
config := m.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
m.Server.Network = *Network
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = m.Server.NewAttrStore
m.Config = config
// Run new program.
if err := m.Run(); err != nil {
return err
}
return nil
}
// RunWithTransport runs Main and returns the dynamically allocated gossip port.
func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (seed string, err error) {
defer close(m.Started)
/*
TEST:
- SetupServer (just static settings from config)
- OpenListener (sets Server.Name to use in gossip)
- NewTransport (gossip)
- SetupNetworking (does the gossip or static stuff) - uses Server.Name
- Open server
PRODUCTION:
- SetupServer (just static settings from config)
- SetupNetworking (does the gossip or static stuff) - calls NewTransport
- Open server - calls OpenListener
*/
// SetupServer
err = m.SetupServer()
if err != nil {
return seed, err
}
// Open server listener.
err = m.Server.OpenListener()
if err != nil {
return seed, err
}
// Open gossip transport to use in SetupServer.
transport, err := gossip.NewTransport(host, bindPort)
if err != nil {
return seed, err
}
m.GossipTransport = transport
if len(joinSeeds) != 0 {
m.Config.Gossip.Seeds = joinSeeds
} else {
m.Config.Gossip.Seeds = []string{transport.URI.String()}
}
seed = transport.URI.String()
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
return seed, err
}
if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil {
return seed, err
}
m.Server.Cluster.Static = false
// Initialize server.
err = m.Server.Open()
if err != nil {
return seed, err
}
return seed, nil
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
panic(err)
}
return client
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// CreateDefinition.
func (m *Main) CreateDefinition(index, def, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
func (m *Main) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
if resp.StatusCode != 204 {
return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return nil
}
////////////////////////////////////////////////////////////////////////////////////
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.
func MustDo(method, urlStr string, body string) *httpResponse {
req, err := http.NewRequest(method, urlStr, strings.NewReader(body))
if err != nil {
panic(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
return &httpResponse{Response: resp, Body: string(buf)}
}
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*http.Response
Body string
}

35
test/pilosa_test.go
View file

@ -15,28 +15,35 @@
package test_test
import (
"encoding/json"
"net/http"
"testing"
"encoding/json"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestNewCluster(t *testing.T) {
cluster := test.MustNewServerCluster(t, 3)
response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status")
numNodes := 3
cluster := test.MustRunMainWithCluster(t, numNodes)
coordinator := cluster[0].Server.Cluster.Coordinator
for i := 1; i < numNodes; i++ {
if coordi := cluster[i].Server.Cluster.Coordinator; coordi != coordinator {
t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator)
}
}
response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status")
if err != nil {
t.Fatalf("getting schema: %v", err)
}
dec := json.NewDecoder(response.Body)
body := struct {
Status struct {
Nodes []struct {
Host string
Schema string
State string
}
State string
Nodes []struct {
Scheme string
Host string
Port int
}
}{}
@ -50,13 +57,11 @@ func TestNewCluster(t *testing.T) {
t.Fatalf("encoding: %v", err)
}
if len(body.Status.Nodes) != 3 {
if len(body.Nodes) != 3 {
t.Fatalf("wrong number of nodes in status: %s", bytes)
}
for i, node := range body.Status.Nodes {
if node.State != "UP" {
t.Fatalf("node %d should be up but is %s", i, node.State)
}
if body.State != pilosa.ClusterStateNormal {
t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State)
}
}

108
uri.go
View file

@ -15,11 +15,14 @@
package pilosa
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/pilosa/pilosa/internal"
)
var schemeRegexp = regexp.MustCompile("^[+a-z]+$")
@ -40,9 +43,9 @@ var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0-
// localhost
// :10101
type URI struct {
scheme string
host string
port uint16
scheme string `json:"scheme"`
host string `json:"host"`
port uint16 `json:"port"`
}
// DefaultURI creates and returns the default URI.
@ -54,6 +57,16 @@ func DefaultURI() *URI {
}
}
type URIs []URI
func (u URIs) HostPortStrings() []string {
s := make([]string, len(u))
for i, a := range u {
s[i] = a.HostPort()
}
return s
}
// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
uri := DefaultURI()
@ -67,11 +80,7 @@ func NewURIFromHostPort(host string, port uint16) (*URI, error) {
// NewURIFromAddress parses the passed address and returns a URI.
func NewURIFromAddress(address string) (*URI, error) {
uri, err := parseAddress(address)
if err != nil {
return nil, err
}
return uri, err
return parseAddress(address)
}
// Scheme returns the scheme of this URI.
@ -144,9 +153,7 @@ func (u URI) Equals(other *URI) bool {
if other == nil {
return false
}
return u.scheme == other.scheme &&
u.host == other.host &&
u.port == other.port
return u == *other
}
// Path returns URI with path
@ -198,3 +205,82 @@ func parseAddress(address string) (uri *URI, err error) {
}
return uri, nil
}
// Encode converts o into its internal representation.
func (u URI) Encode() *internal.URI {
return encodeURI(u)
}
func encodeURI(u URI) *internal.URI {
return &internal.URI{
Scheme: u.scheme,
Host: u.host,
Port: uint32(u.port),
}
}
func DecodeURI(i *internal.URI) URI {
return decodeURI(i)
}
func decodeURI(i *internal.URI) URI {
if i == nil {
return URI{}
}
return URI{
scheme: i.Scheme,
host: i.Host,
port: uint16(i.Port),
}
}
func encodeURIs(a []URI) []*internal.URI {
if len(a) == 0 {
return nil
}
other := make([]*internal.URI, len(a))
for i := range a {
other[i] = encodeURI(a[i])
}
return other
}
func decodeURIs(a []*internal.URI) []URI {
if len(a) == 0 {
return nil
}
other := make([]URI, len(a))
for i := range a {
other[i] = decodeURI(a[i])
}
return other
}
// MarshalJSON marshals URI into a JSON-encoded byte slice.
func (u *URI) MarshalJSON() ([]byte, error) {
var output struct {
Scheme string `json:"scheme,omitempty"`
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
output.Scheme = u.scheme
output.Host = u.host
output.Port = u.port
return json.Marshal(output)
}
func (u *URI) UnmarshalJSON(b []byte) error {
var input struct {
Scheme string `json:"scheme,omitempty"`
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
if err := json.Unmarshal(b, &input); err != nil {
return err
}
u.scheme = input.Scheme
u.host = input.Host
u.port = input.Port
return nil
}

42
view.go
View file

@ -18,6 +18,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
@ -62,7 +63,7 @@ type View struct {
broadcaster Broadcaster
stats StatsClient
RowAttrStore *AttrStore
RowAttrStore AttrStore
LogOutput io.Writer
}
@ -98,6 +99,12 @@ func (v *View) Path() string { return v.path }
// Open opens and initializes the view.
func (v *View) Open() error {
// Never keep a cache for field views.
if strings.HasPrefix(v.name, ViewFieldPrefix) {
v.cacheType = CacheTypeNone
}
if err := func() error {
// Ensure the view's path exists.
if err := os.MkdirAll(v.path, 0777); err != nil {
@ -119,6 +126,9 @@ func (v *View) Open() error {
return nil
}
// logger returns a logger instance for the view.
func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) }
// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
@ -270,6 +280,36 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
return frag
}
// DeleteFragment removes the fragment from the view.
func (v *View) DeleteFragment(slice uint64) error {
fragment := v.fragments[slice]
if fragment == nil {
return ErrFragmentNotFound
}
v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return err
}
// Delete fragment file.
if err := os.Remove(fragment.Path()); err != nil {
return err
}
// Delete fragment cache file.
if err := os.Remove(fragment.CachePath()); err != nil {
v.logger().Printf("no cache file to delete for slice %d", slice)
}
delete(v.fragments, slice)
return nil
}
// SetBit sets a bit within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth

45
view_test.go
View file

@ -17,6 +17,7 @@ package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
@ -25,22 +26,21 @@ import (
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
RowAttrStore *test.AttrStore
RowAttrStore pilosa.AttrStore
}
// NewView returns a new instance of View with a temporary path.
func NewView(index, frame, name string) *View {
file, err := ioutil.TempFile("", "pilosa-view-")
path, err := ioutil.TempDir("", "pilosa-view-")
if err != nil {
panic(err)
}
file.Close()
v := &View{
View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize),
View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize),
RowAttrStore: test.MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore
return v
}
@ -68,7 +68,7 @@ func (v *View) Reopen() error {
}
v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View.RowAttrStore = v.RowAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore
if err := v.Open(); err != nil {
return err
}
@ -93,3 +93,36 @@ func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
}
}
}
// Ensure view can open and retrieve a fragment.
func TestView_DeleteFragment(t *testing.T) {
v := MustOpenView("i", "f", "v")
defer v.Close()
slice := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.DeleteFragment(slice)
if err != nil {
t.Fatal(err)
}
if v.Fragment(slice) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same slice, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {
t.Fatal("failed to create new fragment")
}
}