scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 08:35:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #904 from pilosa/cluster-resize

Browse source 
WIP: Cluster Resize
This commit is contained in:
Travis Turner 2018-03-30 16:12:33 -05:00 • committed by GitHub
commit fb3a2b8b03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
71 changed files with 10733 additions and 3060 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.travis.yml
View file

@ -8,7 +8,7 @@ env:
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
install:
- make vendor
- make vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
@ -19,10 +19,11 @@ before_deploy:
- pip install awscli --user `whoami`
deploy:
- provider: script
script: make prerelease-upload
script: make prerelease-upload GOOS=linux GOARCH=amd64
skip_cleanup: true
on:
branch: master
all_branches: true
go: 1.9
matrix:
allow_failures:
- go: master

59
CHANGELOG.md
View file

@ -5,6 +5,65 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
### Added
- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908))
- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088))
- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133))
- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121))
- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138))
- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132))
- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109))
- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064))
- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063))
- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048))
- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014))
- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013))
- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985))
- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963))
- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972))
### Changed
- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163))
- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166))
- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158))
- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148))
- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797))
- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115))
- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099))
- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077))
- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032))
- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027))
- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001))
- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950))
- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991))
- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907))
### Removed
- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171))
- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142))
- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130))
### Fixed
- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143))
- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139))
- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110))
- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102))
- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105))
- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101))
- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084))
- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979))
- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980))
### Performance
- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))
## [0.8.8] - 2018-02-19
This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions.

6
Gopkg.lock generated
View file

@ -205,12 +205,6 @@
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
name = "github.com/sony/gobreaker"
packages = ["."]
revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36"
version = "0.3.0"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"

17
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test
.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test
DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
@ -11,6 +11,9 @@ PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
BUILD_TIME=`date -u +%FT%T%z`
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
DOCKER_GOLANG_IMAGE=golang:latest
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH))
BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH)
default: test pilosa
@ -75,16 +78,16 @@ else
endif
prerelease-build: vendor
make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa"
cp LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH)
tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/
@echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz"
make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa"
cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/
@echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz"
prerelease:
make prerelease-build GOOS=linux GOARCH=amd64
prerelease-upload: prerelease
aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read
aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read
install: vendor
go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
@ -100,7 +103,7 @@ generate-protoc: .protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
generate-statik: statik
go generate github.com/pilosa/pilosa
go generate github.com/pilosa/pilosa/statik
generate: generate-protoc generate-statik

513
attr.go
View file

@ -16,23 +16,12 @@ package pilosa
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// Attribute data type enum.
const (
AttrTypeString = 1
@ -41,313 +30,111 @@ const (
AttrTypeFloat = 4
)
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
// AttrStore represents an interface for handling row/column attributes.
type AttrStore interface {
Path() string
Open() error
Close() error
Attrs(id uint64) (m map[string]interface{}, err error)
SetAttrs(id uint64, m map[string]interface{}) error
SetBulkAttrs(m map[uint64]map[string]interface{}) error
Blocks() ([]AttrBlock, error)
BlockData(i uint64) (map[uint64]map[string]interface{}, error)
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
func init() {
NopAttrStore = &nopAttrStore{}
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
// NopAttrStore represents an AttrStore that doesn't do anything.
var NopAttrStore AttrStore
func NewNopAttrStore(string) AttrStore {
return &nopAttrStore{}
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// nopAttrStore represents a no-op implementation of the AttrStore interface.
type nopAttrStore struct{}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) *AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
// Path is a no-op implementation of AttrStore Path method.
func (s *nopAttrStore) Path() string { return "" }
// Open is a no-op implementation of AttrStore Open method.
func (s *nopAttrStore) Open() error {
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
// Close is a no-op implementation of AttrStore Close method.
func (s *nopAttrStore) Close() error {
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
// Attrs is a no-op implementation of AttrStore Attrs method.
func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return nil, nil
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
return nil
}
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
// Blocks is a no-op implementation of AttrStore Blocks method.
func (s *nopAttrStore) Blocks() ([]AttrBlock, error) {
return nil, nil
}
// BlockData is a no-op implementation of AttrStore BlockData method.
func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []AttrBlock
for cur.nextBlock() {
block := AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
m[btou64(k)] = decodeAttrs(pb.GetAttrs())
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
a, other = a[1:], other[1:]
}
}
// Marshal and save new values.
buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
func encodeAttrs(m map[string]interface{}) []*internal.Attr {
@ -421,136 +208,16 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} {
return other
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
// EncodeAttrs encodes an attribute map into a byte slice.
func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
a, other = a[1:], other[1:]
}
}
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
// DecodeAttrs decodes a byte slice into an attribute map.
func DecodeAttrs(v []byte) (map[string]interface{}, error) {
var pb internal.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}

465
boltdb/attrstore.go Normal file
View file

@ -0,0 +1,465 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// AttrCache represents a cache for attributes.
type AttrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &AttrStore{
path: path,
attrCache: NewAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return nil
}
// Close closes the store.
func (s *AttrStore) Close() error {
if s.db != nil {
s.db.Close()
}
return nil
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
if err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// Add to cache.
s.attrCache.Set(id, m)
return
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return err
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Sort(uint64Slice(ids))
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
// Iterate over each block.
var blocks []pilosa.AttrBlock
for cur.nextBlock() {
block := pilosa.AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
tx, err := s.db.Begin(false)
if err != nil {
return nil, err
}
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
return nil, err
}
m[btou64(k)] = attrs
}
return m, nil
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
return pilosa.DecodeAttrs(v)
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
}
}
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
return nil, err
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, err
}
return attr, nil
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}

124
broadcast.go
View file

@ -22,38 +22,27 @@ import (
"github.com/pilosa/pilosa/internal"
)
// NodeSet represents an interface for Node membership and inter-node communication.
type NodeSet interface {
// Returns a list of all Nodes in the cluster
Nodes() []*Node
// Open starts any network activity implemented by the NodeSet
Open() error
// MemberSet represents an interface for Node membership and inter-node communication.
type MemberSet interface {
// Open starts any network activity implemented by the MemberSet
// Node is the local node, used for membership broadcasts.
Open(n *Node) error
}
// StaticNodeSet represents a basic NodeSet for testing.
type StaticNodeSet struct {
// StaticMemberSet represents a basic MemberSet for testing.
type StaticMemberSet struct {
nodes []*Node
}
// NewStaticNodeSet creates a statically defined NodeSet.
func NewStaticNodeSet() *StaticNodeSet {
return &StaticNodeSet{}
// NewStaticMemberSet creates a statically defined MemberSet.
func NewStaticMemberSet(nodes []*Node) *StaticMemberSet {
return &StaticMemberSet{
nodes: nodes,
}
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (s *StaticNodeSet) Nodes() []*Node {
return s.nodes
}
// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing.
func (s *StaticNodeSet) Open() error {
return nil
}
// Join sets the NodeSet nodes to the slice of Nodes passed in.
func (s *StaticNodeSet) Join(nodes []*Node) error {
s.nodes = nodes
// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing.
func (s *StaticMemberSet) Open(n *Node) error {
return nil
}
@ -61,6 +50,7 @@ func (s *StaticNodeSet) Join(nodes []*Node) error {
type Broadcaster interface {
SendSync(pb proto.Message) error
SendAsync(pb proto.Message) error
SendTo(to *Node, pb proto.Message) error
}
func init() {
@ -73,16 +63,21 @@ var NopBroadcaster Broadcaster
type nopBroadcaster struct{}
// SendSync A no-op implemenetation of Broadcaster SendSync method.
// SendSync A no-op implementation of Broadcaster SendSync method.
func (n *nopBroadcaster) SendSync(pb proto.Message) error {
return nil
}
// SendAsync A no-op implemenetation of Broadcaster SendAsync method.
// SendAsync A no-op implementation of Broadcaster SendAsync method.
func (n *nopBroadcaster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error {
return nil
}
// BroadcastHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type BroadcastHandler interface {
@ -117,23 +112,32 @@ var NopGossiper Gossiper
type nopGossiper struct{}
// SendAsync A no-op implemenetation of Gossiper SendAsync method.
// SendAsync A no-op implementation of Gossiper SendAsync method.
func (n *nopGossiper) SendAsync(pb proto.Message) error {
return nil
}
// Broadcast message types.
const (
MessageTypeCreateSlice = 1
MessageTypeCreateIndex = 2
MessageTypeDeleteIndex = 3
MessageTypeCreateFrame = 4
MessageTypeDeleteFrame = 5
MessageTypeCreateInputDefinition = 6
MessageTypeDeleteInputDefinition = 7
MessageTypeDeleteView = 8
MessageTypeCreateField = 9
MessageTypeDeleteField = 10
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateFrame
MessageTypeDeleteFrame
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateInputDefinition
MessageTypeDeleteInputDefinition
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -150,6 +154,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
typ = MessageTypeDeleteFrame
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
case *internal.DeleteFieldMessage:
@ -158,8 +166,22 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateInputDefinition
case *internal.DeleteInputDefinitionMessage:
typ = MessageTypeDeleteInputDefinition
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -186,6 +208,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:
m = &internal.DeleteFrameMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
@ -194,8 +220,22 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateInputDefinitionMessage{}
case MessageTypeDeleteInputDefinition:
m = &internal.DeleteInputDefinitionMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
}

38
client.go
View file

@ -89,9 +89,6 @@ func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[st
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.clientURI(ctx), "/slices/max")
u.RawQuery = (&url.Values{
"inverse": {strconv.FormatBool(inverse)},
}).Encode()
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
@ -108,14 +105,17 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
}
defer resp.Body.Close()
var rsp sliceMaxResponse
var rsp getSlicesMaxResponse
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
return rsp.MaxSlices, nil
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
// Schema returns all index and frame schema information.
@ -296,7 +296,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
// Import to each node.
for _, node := range nodes {
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -317,13 +317,12 @@ func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, b
}
node := &Node{
Scheme: c.defaultURI.Scheme(),
Host: c.defaultURI.HostPort(),
URI: *c.defaultURI,
}
// Import to node.
if err := c.importNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
return nil
@ -448,7 +447,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel
// Import to each node.
for _, node := range nodes {
if err := c.importValueNode(ctx, node, buf); err != nil {
return fmt.Errorf("import node: host=%s, err=%s", node.Host, err)
return fmt.Errorf("import node: host=%s, err=%s", node.URI, err)
}
}
@ -536,7 +535,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
return nil
@ -688,6 +687,13 @@ func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, frame, view, slice, node)
}
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
@ -717,7 +723,7 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
return nil, ErrFragmentNotFound
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.Host, resp.StatusCode)
return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
return resp.Body, nil
@ -796,7 +802,7 @@ func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, i
// Return error if response not OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.Host, resp.StatusCode)
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
}
@ -1130,6 +1136,8 @@ func (c *InternalHTTPClient) clientURI(ctx context.Context) *URI {
clientURI := c.defaultURI
if contextURI, ok := ctx.Value("uri").(*URI); ok {
clientURI = contextURI
} else if contextURI, ok := ctx.Value("uri").(URI); ok {
clientURI = &contextURI
}
return clientURI
}
@ -1318,8 +1326,8 @@ func uriPathToURL(uri *URI, path string) url.URL {
func nodePathToURL(node *Node, path string) url.URL {
return url.URL{
Scheme: node.Scheme,
Host: node.Host,
Scheme: node.URI.Scheme(),
Host: node.URI.HostPort(),
Path: path,
}
}

43
client_test.go
View file

@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
for i := 0; i < numNodes; i++ {
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
server[i].Handler.URI = server[i].HostURI()
server[i].Handler.Cluster = c
server[i].Handler.Cluster.Nodes[i].Host = server[i].Host()
server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI()
server[i].Handler.Holder = hldr[i].Holder
server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i]
}
return server, hldr
}
@ -64,24 +64,21 @@ func TestClient_MultiNode(t *testing.T) {
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[0].Holder
e.Scheme = cluster.Nodes[0].Scheme
e.Host = cluster.Nodes[0].Host
e.Node = cluster.Nodes[0]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[1].Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
e.Holder = hldr[2].Holder
e.Scheme = cluster.Nodes[2].Scheme
e.Host = cluster.Nodes[2].Host
e.Node = cluster.Nodes[2]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
@ -89,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host())
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
if ownNum == num {
@ -220,10 +217,10 @@ func TestClient_Import(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -271,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -320,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
@ -358,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -423,10 +420,10 @@ func TestClient_BackupInverseView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -460,10 +457,10 @@ func TestClient_BackupInvalidView(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
c := test.MustNewClient(s.Host(), defaultClient)
@ -489,10 +486,10 @@ func TestClient_FragmentBlocks(t *testing.T) {
s := test.NewServer()
defer s.Close()
s.Handler.URI = s.HostURI()
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.Holder = hldr.Holder
s.Handler.Node = s.Handler.Cluster.Nodes[0]
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)

1731
cluster.go
View file

File diff suppressed because it is too large Load diff

309
cluster_internal_test.go Normal file
View file

@ -0,0 +1,309 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"io/ioutil"
"reflect"
"testing"
"github.com/pilosa/pilosa/internal"
)
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
c := NewCluster()
c.addNodeBasicSorted(node0)
c.addNodeBasicSorted(node1)
tests := []struct {
idx string
maxSlice uint64
frameViews viewsByFrame
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
},
},
{
idx: "foo",
maxSlice: uint64(3),
frameViews: viewsByFrame{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
},
},
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(name string) *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, name)
if err != nil {
panic(err)
}
return index
}
// Ensure that fragSources creates the correct fragment mapping.
func TestFragSources(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
uri3, err := NewURIFromAddress("host3")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node3 := &Node{ID: "node3", URI: *uri3}
c1 := NewCluster()
c1.ReplicaN = 1
c1.addNodeBasicSorted(node0)
c1.addNodeBasicSorted(node1)
c2 := NewCluster()
c2.ReplicaN = 1
c2.addNodeBasicSorted(node0)
c2.addNodeBasicSorted(node1)
c2.addNodeBasicSorted(node2)
c3 := NewCluster()
c3.ReplicaN = 2
c3.addNodeBasicSorted(node0)
c3.addNodeBasicSorted(node1)
c4 := NewCluster()
c4.ReplicaN = 2
c4.addNodeBasicSorted(node0)
c4.addNodeBasicSorted(node1)
c4.addNodeBasicSorted(node2)
c5 := NewCluster()
c5.ReplicaN = 2
c5.addNodeBasicSorted(node0)
c5.addNodeBasicSorted(node1)
c5.addNodeBasicSorted(node2)
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 1300000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 2600000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 3900000, nil)
if err != nil {
t.Fatal(err)
}
tests := []struct {
from *Cluster
to *Cluster
idx *Index
expected map[string][]*internal.ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{},
"node1": []*internal.ResizeSource{},
"node2": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c4,
to: c3,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
},
{
from: c5,
to: c4,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
},
"node2": []*internal.ResizeSource{},
},
err: "",
},
{
from: c2,
to: c4,
idx: idx,
expected: nil,
err: "clusters are the same size",
},
{
from: c1,
to: c5,
idx: idx,
expected: nil,
err: "adding more than one node at a time is not supported",
},
{
from: c5,
to: c1,
idx: idx,
expected: nil,
err: "removing more than one node at a time is not supported",
},
}
for _, test := range tests {
actual, err := (test.from).fragSources(test.to, test.idx)
if test.err != "" {
if err.Error() != test.err {
t.Fatalf("expected error: %s", test.err)
}
} else {
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
}
}
}
// Ensure that fragSources creates the correct fragment mapping.
func TestResizeJob(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
tests := []struct {
existingNodes []*Node
node *Node
action string
expectedIDs map[string]bool
}{
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual.IDs, test.expectedIDs) {
t.Errorf("expected: %v, but got: %v", test.expectedIDs, actual.IDs)
}
}
}

481
cluster_test.go
View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"bytes"
"math/rand"
"reflect"
"testing"
@ -29,9 +30,9 @@ import (
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
{URI: test.NewURIFromHostPort("serverA", 1000)},
{URI: test.NewURIFromHostPort("serverB", 1000)},
{URI: test.NewURIFromHostPort("serverC", 1000)},
},
Hasher: test.NewModHasher(),
ReplicaN: 2,
@ -91,52 +92,442 @@ func TestHasher(t *testing.T) {
}
}
// Ensure that an empty cluster returns a valid (empty) NodeSet
func TestCluster_NodeSetHosts(t *testing.T) {
c := pilosa.Cluster{}
if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) {
t.Fatalf("unexpected slice of hosts: %s", h)
}
}
// Ensure cluster can compare its Nodes and Members
func TestCluster_NodeStates(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
NodeSet: &pilosa.StaticNodeSet{},
}
err := c.NodeSet.(*pilosa.StaticNodeSet).Join([]*pilosa.Node{
&pilosa.Node{Host: "serverA:1000"},
&pilosa.Node{Host: "serverC:1000"},
&pilosa.Node{Host: "serverD:1000"},
})
if err != nil {
t.Fatalf("unexpected gossiper nodes: %s", err)
}
// Verify a DOWN node is reported, and extraneous nodes are ignored
if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{
"serverA:1000": pilosa.NodeStateUp,
"serverB:1000": pilosa.NodeStateDown,
"serverC:1000": pilosa.NodeStateUp,
}) {
t.Fatalf("unexpected node state: %s", spew.Sdump(a))
}
}
// Ensure OwnsSlices can find the actual slice list for node and index
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := test.NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := test.NewCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := test.NewURIFromHostPort("node0", 0)
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
uri3 := test.NewURIFromHostPort("node3", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
node3 := &pilosa.Node{ID: "node3", URI: uri3}
nodes := []*pilosa.Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := pilosa.Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs()
expected := []pilosa.URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs()
expected := []pilosa.URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := pilosa.Nodes(nodes).Contains(node1)
actualFalse := pilosa.Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := pilosa.Nodes(nodes).Clone()
actual := pilosa.Nodes(clone).URIs()
expected := []pilosa.URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := test.NewURIFromHostPort("node1", 0)
uri2 := test.NewURIFromHostPort("node2", 0)
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
c1 := *pilosa.NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *pilosa.NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"}
uri0 := test.NewURIFromHostPort("host0", 0)
uri1 := test.NewURIFromHostPort("host1", 0)
uri2 := test.NewURIFromHostPort("host2", 0)
invalid := test.NewURIFromHostPort("invalid", 0)
node0 := &pilosa.Node{ID: "node0", URI: uri0}
node1 := &pilosa.Node{ID: "node1", URI: uri1}
node2 := &pilosa.Node{ID: "node2", URI: uri2}
nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := test.NewTestCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != pilosa.ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &pilosa.Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != pilosa.ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node2.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := test.NewTestCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{
InverseEnabled: false,
RangeEnabled: true,
//CacheType: pilosa.CacheTypeNone,
Fields: []*pilosa.Field{
{
Name: "fld0",
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
},
},
}); err != nil {
t.Fatal(err)
}
tc.SetFieldValue("i", "fields", 1, "fld0", -10)
tc.SetFieldValue("i", "fields", 1, "fld0", 10)
tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Frame := node0.Holder.Frame("i", "f")
node0View := node0Frame.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
node0Frame = node0.Holder.Frame("i", "fields")
node0View = node0Frame.View("field_fld0")
node0Fragment = node0View.Fragment(1)
node0ChecksumFld := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State())
} else if node1.State() != pilosa.ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State())
}
expectedTop := &pilosa.Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Values
// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
node1Frame = node1.Holder.Frame("i", "fields")
node1View = node1Frame.View("field_fld0")
node1Fragment = node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := test.NewCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

1
cmd/root_test.go
View file

@ -181,7 +181,6 @@ func TestRootCommand_Config(t *testing.T) {
bind = "127.0.0.1:10101"
[cluster]
poll-interval = "2m0s"
replicas = 2
partitions = 128
hosts = [

12
cmd/server_test.go
View file

@ -45,15 +45,14 @@ func TestServerConfig(t *testing.T) {
// TEST 0
{
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
bind = "localhost:0"
max-writes-per-request = 3000
[cluster]
poll-interval = "45s"
type = "static"
disabled = true
replicas = 2
hosts = [
"localhost:19444",
@ -66,7 +65,6 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.Bind, "localhost:10111")
v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
return v.Error()
@ -80,7 +78,7 @@ func TestServerConfig(t *testing.T) {
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
[cluster]
type = "static"
disabled = true
hosts = [
"localhost:19444",
]
@ -94,13 +92,12 @@ func TestServerConfig(t *testing.T) {
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.type", "static"},
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"},
cfgFileContent: `
bind = "localhost:19444"
data-dir = "` + actualDataDir + `"
[cluster]
poll-interval = "2m0s"
hosts = [
"localhost:19444",
]
@ -116,7 +113,6 @@ func TestServerConfig(t *testing.T) {
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Minute*2))
v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11))
v.Check(cmd.Server.CPUProfile, profFile.Name())
v.Check(cmd.Server.CPUTime, time.Minute)

199
config.go
View file

@ -26,27 +26,92 @@ const (
)
const (
// DefaultDataDir is the default data directory.
DefaultDataDir = "~/.pilosa"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultPort is the default port use with the hostname.
// DefaultPort is the default port to use with the hostname.
DefaultPort = "10101"
// DefaultClusterType sets the node intercommunication method.
DefaultClusterType = ClusterGossip
// DefaultGossipPort indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// DefaultClusterDisabled sets the node intercommunication method.
DefaultClusterDisabled = false
// DefaultMetrics sets the internal metrics to no-op.
DefaultMetrics = "nop"
// DefaultMaxWritesPerRequest is the default number of writes per request.
DefaultMaxWritesPerRequest = 5000
)
// ClusterTypes set of cluster types.
var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterGossip}
// Gossip config based on memberlist.Config.
// Port indicates the port to which pilosa should bind for internal state sharing.
DefaultGossipPort = "14000"
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
DefaultGossipStreamTimeout = 10 * time.Second
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
DefaultGossipSuspicionMult = 4
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
DefaultGossipPushPullInterval = 30 * time.Second
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
DefaultGossipProbeInterval = 1 * time.Second
DefaultGossipProbeTimeout = 500 * time.Millisecond
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
DefaultGossipInterval = 200 * time.Millisecond
DefaultGossipNodes = 3
DefaultGossipToTheDeadTime = 30 * time.Second
DefaultMetricPollInterval = 0 * time.Minute
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
@ -62,28 +127,6 @@ type TLSConfig struct {
type Config struct {
DataDir string `toml:"data-dir"`
Bind string `toml:"bind"`
// GossipPort DEPRECATED
GossipPort string `toml:"gossip-port"`
// GossipSeed DEPRECATED
GossipSeed string `toml:"gossip-seed"`
Gossip struct {
Port string `toml:"port"`
Seed string `toml:"seed"`
Key string `toml:"key"`
} `toml:"gossip"`
Cluster struct {
ReplicaN int `toml:"replicas"`
Type string `toml:"type"`
Hosts []string `toml:"hosts"`
PollInterval Duration `toml:"poll-interval"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
// Limits the number of mutating commands that can be in a single request to
// the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs.
@ -91,67 +134,93 @@ type Config struct {
LogPath string `toml:"log-path"`
// TLS
TLS TLSConfig
Cluster struct {
Disabled bool `toml:"disabled"`
Coordinator bool `toml:"coordinator"`
ReplicaN int `toml:"replicas"`
Hosts []string `toml:"hosts"`
LongQueryTime Duration `toml:"long-query-time"`
} `toml:"cluster"`
Gossip struct {
Port string `toml:"port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
StreamTimeout Duration `toml:"stream-timeout"`
SuspicionMult int `toml:"suspicion-mult"`
PushPullInterval Duration `toml:"push-pull-interval"`
ProbeTimeout Duration `toml:"probe-timeout"`
ProbeInterval Duration `toml:"probe-interval"`
Nodes int `toml:"nodes"`
Interval Duration `toml:"interval"`
ToTheDeadTime Duration `toml:"to-the-dead-time"`
} `toml:"gossip"`
AntiEntropy struct {
Interval Duration `toml:"interval"`
} `toml:"anti-entropy"`
Metric struct {
Service string `toml:"service"`
Host string `toml:"host"`
PollInterval Duration `toml:"poll-interval"`
Diagnostics bool `toml:"diagnostics"`
} `toml:"metric"`
TLS TLSConfig
}
// NewConfig returns an instance of Config with default options.
func NewConfig() *Config {
c := &Config{
Bind: DefaultHost + ":" + DefaultPort,
DataDir: DefaultDataDir,
Bind: ":" + DefaultPort,
MaxWritesPerRequest: DefaultMaxWritesPerRequest,
// LogPath: "",
TLS: TLSConfig{},
}
// Cluster config.
c.Cluster.Disabled = DefaultClusterDisabled
// c.Cluster.Coordinator = false
c.Cluster.ReplicaN = DefaultReplicaN
c.Cluster.Type = DefaultClusterType
c.Cluster.PollInterval = Duration(DefaultPollingInterval)
c.Cluster.Hosts = []string{}
c.Cluster.LongQueryTime = Duration(time.Minute)
// Gossip config.
// c.Gossip.Port = ""
// c.Gossip.Seeds = []string{}
// c.Gossip.Key = ""
c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout)
c.Gossip.SuspicionMult = DefaultGossipSuspicionMult
c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval)
c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout)
c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval)
c.Gossip.Nodes = DefaultGossipNodes
c.Gossip.Interval = Duration(DefaultGossipInterval)
c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime)
// AntiEntropy config.
c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval)
// Metric config.
c.Metric.Service = DefaultMetrics
// c.Metric.Host = ""
c.Metric.PollInterval = Duration(DefaultMetricPollInterval)
c.Metric.Diagnostics = true
c.TLS = TLSConfig{}
return c
}
// Validate that all configuration permutations are compatible with each other.
func (c *Config) Validate() error {
if !StringInSlice(c.Cluster.Type, ClusterTypes) {
return ErrConfigClusterTypeInvalid
if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 {
return ErrConfigClusterEnabledHosts
}
if c.Cluster.Type == ClusterGossip {
if len(c.Cluster.Hosts) > 0 {
bindWithDefaults, err := AddressWithDefaults(c.Bind)
if err != nil {
return err
}
if !c.foundHost(bindWithDefaults) {
return ErrConfigHostsMissing
}
}
}
return nil
}
func (c *Config) foundHost(host *URI) bool {
for _, clusterHost := range c.Cluster.Hosts {
uri, err := NewURIFromAddress(clusterHost)
if err != nil {
continue
}
if host.Equals(uri) {
return true
}
}
return false
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration

24
config_test.go
View file

@ -25,27 +25,15 @@ import (
func Test_NewConfig(t *testing.T) {
c := pilosa.NewConfig()
if c.Cluster.Disabled != pilosa.DefaultClusterDisabled {
t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled)
}
// Ensure that hosts can't be specificed on a non-disabled cluster.
c.Cluster.Hosts = []string{c.Bind, "localhost:10102"}
// Change cluster type from the default (gossip) to an invalid string.
c.Cluster.Type = "invalid-type"
if err := c.Validate(); err != pilosa.ErrConfigClusterTypeInvalid {
t.Fatal(err)
}
// Change cluster type back to gossip.
c.Cluster.Type = pilosa.ClusterGossip
// Check for bind address in cluster hosts.
c.Bind = "localhost:1"
if err := c.Validate(); err != pilosa.ErrConfigHostsMissing {
t.Fatal(err)
}
c.Bind = "localhost:10101"
c.Cluster.ReplicaN = 2
c.GossipSeed = "localhost:14000"
if err := c.Validate(); err != nil {
if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts {
t.Fatal(err)
}
}

6
ctl/backup_test.go
View file

@ -50,9 +50,11 @@ func TestBackupCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")

7
ctl/config_test.go
View file

@ -17,11 +17,12 @@ package ctl
import (
"bytes"
"context"
"github.com/pilosa/pilosa"
"io"
"os"
"strings"
"testing"
"github.com/pilosa/pilosa"
)
func TestConfigCommand_Run(t *testing.T) {
@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) {
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), pilosa.DefaultHost) {
t.Fatalf("Unexpected config: %s", buf.String())
} else if !strings.Contains(buf.String(), ":10101") {
t.Fatalf("Unexpected config: \n%s", buf.String())
}
}

6
ctl/export_test.go
View file

@ -63,9 +63,11 @@ func TestExportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()

1
ctl/generate_config.go
View file

@ -43,7 +43,6 @@ bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
poll-interval = "2m0s"
replicas = 1
hosts = [
"localhost:10101",

12
ctl/import_test.go
View file

@ -69,9 +69,11 @@ func TestImportCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()
@ -109,9 +111,11 @@ func TestImportCommand_RunValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0] = node
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()

6
ctl/restore_test.go
View file

@ -52,9 +52,11 @@ func TestRestoreCommand_Run(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s.Handler.URI = uri
node := &pilosa.Node{ID: "node", URI: *uri}
s.Handler.Node = node
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)

52
ctl/server.go
View file

@ -24,26 +24,44 @@ import (
// BuildServerFlags attaches a set of flags to the command for a server instance.
func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
// TLS
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
// Cluster
flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this?
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path")
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
// AntiEntropy
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
// Metric
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
// CPU Profiling
flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]")
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify)
}

3
ctl/server_test.go
View file

@ -28,9 +28,6 @@ func TestBuildServerFlags(t *testing.T) {
stdin, stdout, stderr := GetIO(buf)
Server := server.NewCommand(stdin, stdout, stderr)
BuildServerFlags(cm, Server)
if cm.Flags().Lookup("gossip-port").Name == "" {
t.Fatal("gossip-port flag is required")
}
if cm.Flags().Lookup("data-dir").Name == "" {
t.Fatal("data-dir flag is required")
}

324
diagnostics.go Normal file
View file

@ -0,0 +1,324 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
)
// Default version check URL.
const (
defaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// DiagnosticsCollector represents a collector/sender of diagnostics data.
type DiagnosticsCollector struct {
mu sync.Mutex
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
logOutput io.Writer
server *Server
}
// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
return &DiagnosticsCollector{
host: host,
VersionURL: defaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: &http.Client{Timeout: 10 * time.Second},
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *DiagnosticsCollector) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// Flush sends the current metrics.
func (d *DiagnosticsCollector) Flush() error {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, err := d.encode()
if err != nil {
return err
}
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return err
}
// Intentionally ignoring response body, as user does not need to be notified of error.
defer resp.Body.Close()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *DiagnosticsCollector) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// If version has not changed since the last check, return
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.compareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
}
return nil
}
// compareVersion check version strings.
func (d *DiagnosticsCollector) compareVersion(value string) error {
currentVersion := versionSegments(value)
localVersion := versionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *DiagnosticsCollector) encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *DiagnosticsCollector) Set(name string, value interface{}) {
switch v := value.(type) {
case string:
if v == "" {
// Do not set empty string
return
}
}
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *DiagnosticsCollector) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *DiagnosticsCollector) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// logErr logs the error and returns true if an error exists
func (d *DiagnosticsCollector) logErr(err error) bool {
if err != nil {
d.logOutput.Write([]byte(err.Error()))
return true
}
return false
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.SystemInfo.Uptime()
if !d.logErr(err) {
d.Set("HostUptime", uptime)
}
platform, err := d.server.SystemInfo.Platform()
if !d.logErr(err) {
d.Set("OSPlatform", platform)
}
family, err := d.server.SystemInfo.Family()
if !d.logErr(err) {
d.Set("OSFamily", family)
}
version, err := d.server.SystemInfo.OSVersion()
if !d.logErr(err) {
d.Set("OSVersion", version)
}
kernelVersion, err := d.server.SystemInfo.KernelVersion()
if !d.logErr(err) {
d.Set("OSKernelVersion", kernelVersion)
}
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
memFree, err := d.server.SystemInfo.MemFree()
if !d.logErr(err) {
d.Set("MemFree", memFree)
}
memTotal, err := d.server.SystemInfo.MemTotal()
if !d.logErr(err) {
d.Set("MemTotal", memTotal)
}
memUsed, err := d.server.SystemInfo.MemUsed()
if !d.logErr(err) {
d.Set("MemUsed", memUsed)
}
}
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range d.server.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}
// versionSegments returns the numeric segments of the version as a slice of ints.
func versionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}
// SystemInfo collects information about the host OS.
type SystemInfo interface {
Uptime() (uint64, error)
Platform() (string, error)
Family() (string, error)
OSVersion() (string, error)
KernelVersion() (string, error)
MemFree() (uint64, error)
MemTotal() (uint64, error)
MemUsed() (uint64, error)
}
// NewNopSystemInfo creates a no-op implementation of SystemInfo.
func NewNopSystemInfo() *NopSystemInfo {
return &NopSystemInfo{}
}
// NopSystemInfo is a no-op implementation of SystemInfo.
type NopSystemInfo struct {
}
// Uptime is a no-op implementation of SystemInfo.Uptime.
func (n *NopSystemInfo) Uptime() (uint64, error) {
return 0, nil
}
// Platform is a no-op implementation of SystemInfo.Platform.
func (n *NopSystemInfo) Platform() (string, error) {
return "", nil
}
// Family is a no-op implementation of SystemInfo.Family.
func (n *NopSystemInfo) Family() (string, error) {
return "", nil
}
// OSVersion is a no-op implementation of SystemInfo.OSVersion.
func (n *NopSystemInfo) OSVersion() (string, error) {
return "", nil
}
// KernelVersion is a no-op implementation of SystemInfo.KernelVersion.
func (n *NopSystemInfo) KernelVersion() (string, error) {
return "", nil
}
// MemFree is a no-op implementation of SystemInfo.MemFree.
func (n *NopSystemInfo) MemFree() (uint64, error) {
return 0, nil
}
// MemTotal is a no-op implementation of SystemInfo.MemTotal.
func (n *NopSystemInfo) MemTotal() (uint64, error) {
return 0, nil
}
// MemUsed is a no-op implementation of SystemInfo.MemUsed.
func (n *NopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}

267
diagnostics/diagnostics.go
View file

@ -1,267 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"github.com/sony/gobreaker"
)
// TODO: unique Cluster ID
// Default version check URL.
const (
DefaultVersionCheckURL = "https://diagnostics.pilosa.com/v0/version"
)
type versionResponse struct {
Version string `json:"version"`
Message string `json:"message"`
}
// Diagnostics represents a client to the Pilosa cluster.
type Diagnostics struct {
mu sync.Mutex
wg sync.WaitGroup
closing chan struct{}
host string
VersionURL string
version string
lastVersion string
startTime int64
start time.Time
metrics map[string]interface{}
client *http.Client
interval time.Duration
cb *gobreaker.CircuitBreaker
logOutput io.Writer
}
// New returns a pointer to a new Diagnostics Client given an addr in the format "hostname:port".
func New(host string) *Diagnostics {
return &Diagnostics{
closing: make(chan struct{}),
host: host,
VersionURL: DefaultVersionCheckURL,
startTime: time.Now().Unix(),
start: time.Now(),
client: http.DefaultClient,
metrics: make(map[string]interface{}),
logOutput: ioutil.Discard,
}
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *Diagnostics) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// SetInterval of the diagnostic go routine and match with the circuit breaker timeout.
func (d *Diagnostics) SetInterval(i time.Duration) {
d.interval = i
}
// schedule start the diagnostics service ticker.
func (d *Diagnostics) schedule() {
ticker := time.NewTicker(d.interval)
defer ticker.Stop()
for {
select {
case <-d.closing:
return
case <-ticker.C:
d.CheckVersion()
d.Flush()
}
}
}
// Flush sends the current metrics.
func (d *Diagnostics) Flush() error {
d.mu.Lock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, _ := d.Encode()
d.mu.Unlock()
_, err := d.cb.Execute(func() (interface{}, error) {
req, err := http.NewRequest("POST", d.host, bytes.NewReader(buf))
req.Header.Set("Content-Type", "application/json")
resp, err := d.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// TODO verify response
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
})
return err
}
// Open configures the circuit breaker used by the HTTP client.
func (d *Diagnostics) Open() {
var st gobreaker.Settings
if d.interval > 0 {
st.Timeout = d.interval * 2
}
d.cb = gobreaker.NewCircuitBreaker(st)
d.logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every hour. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics")
}
// Close notify goroutine to stop.
func (d *Diagnostics) Close() error {
close(d.closing)
d.wg.Wait()
return nil
}
// CheckVersion of the local build against Pilosa master.
func (d *Diagnostics) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return fmt.Errorf("json decode: %s", err)
}
// Same a version as last test
if rsp.Version == d.lastVersion {
return nil
}
d.lastVersion = rsp.Version
if err := d.CompareVersion(rsp.Version); err != nil {
d.logger().Printf("%s\n", err.Error())
}
return nil
}
// CompareVersion check version strings.
func (d *Diagnostics) CompareVersion(value string) error {
currentVersion := VersionSegments(value)
localVersion := VersionSegments(d.version)
if localVersion[0] < currentVersion[0] { //Major
return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor
return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value)
} else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch
return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value)
}
return nil
}
// Encode metrics maps into the json message format.
func (d *Diagnostics) Encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *Diagnostics) Set(name string, value interface{}) {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics[name] = value
}
// SetLogger Set the logger output type.
func (d *Diagnostics) SetLogger(logger io.Writer) {
d.logOutput = logger
}
// logger returns a logger that writes to LogOutput.
func (d *Diagnostics) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *Diagnostics) EnrichWithOSInfo() {
osInfo, err := host.Info()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("HostUptime", osInfo.Uptime)
platform, family, version, err := host.PlatformInformation()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSPlatform", platform)
d.Set("OSFamily", family)
d.Set("OSVersion", version)
kernelVersion, err := host.KernelVersion()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSKernelVersion", kernelVersion)
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *Diagnostics) EnrichWithMemoryInfo() {
memory, err := mem.VirtualMemory()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("MemFree", memory.Free)
d.Set("MemTotal", memory.Total)
d.Set("MemUsed", memory.Used)
}
// VersionSegments returns the numeric segments of the version as a slice of ints.
func VersionSegments(segments string) []int {
segments = strings.Trim(segments, "v")
segments = strings.Split(segments, "-")[0]
s := strings.Split(segments, ".")
segmentSlice := make([]int, len(s))
for i, v := range s {
segmentSlice[i], _ = strconv.Atoi(v)
}
return segmentSlice
}

43
diagnostics/diagnostics_test.go → diagnostics_internal_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics_test
package pilosa
import (
"encoding/json"
@ -23,25 +23,20 @@ import (
"runtime"
"strings"
"testing"
"github.com/pilosa/pilosa/diagnostics"
)
func TestDiagnosticsClient(t *testing.T) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
d.Open()
defer d.Close()
d.Set("gg", 10)
d.Set("ss", "ss")
data, err := d.Encode()
data, err := d.encode()
if err != nil {
t.Fatal(err)
}
@ -58,7 +53,7 @@ func TestDiagnosticsClient(t *testing.T) {
// Test the metrics after a flush.
d.Flush()
data, err = d.Encode()
data, err = d.encode()
if err != nil {
t.Fatal(err)
}
@ -74,7 +69,7 @@ func TestDiagnosticsClient(t *testing.T) {
func TestDiagnosticsVersion_Parse(t *testing.T) {
version := "0.1.1"
vs := diagnostics.VersionSegments(version)
vs := versionSegments(version)
output := []int{0, 1, 1}
if !reflect.DeepEqual(vs, output) {
@ -83,35 +78,33 @@ func TestDiagnosticsVersion_Parse(t *testing.T) {
}
func TestDiagnosticsVersion_Compare(t *testing.T) {
d := diagnostics.New("localhost:10101")
d.Open()
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "v0.1.1"
d.SetVersion(version)
err := d.CompareVersion("v1.7.0")
err := d.compareVersion("v1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("1.7.0")
err = d.compareVersion("1.7.0")
if !strings.Contains(err.Error(), "A newer version") {
t.Fatalf("Expected a newer version is available, actual error: %s", err)
}
err = d.CompareVersion("0.7.0")
err = d.compareVersion("0.7.0")
if !strings.Contains(err.Error(), "The latest Minor release is") {
t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.2")
err = d.compareVersion("0.1.2")
if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") {
t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err)
}
err = d.CompareVersion("0.1.1")
err = d.compareVersion("0.1.1")
if err != nil {
t.Fatalf("Versions should match")
}
d.SetVersion("v1.7.0")
err = d.CompareVersion("0.7.2")
err = d.compareVersion("0.7.2")
if err != nil {
t.Fatalf("Local version is greater")
}
@ -125,11 +118,9 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
Version: "1.1.1",
})
}))
defer server.Close()
// Create a new client.
d := diagnostics.New("localhost:10101")
defer d.Close()
d := NewDiagnosticsCollector("localhost:10101")
version := "0.1.1"
d.SetVersion(version)
@ -138,10 +129,6 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
d.CheckVersion()
}
type versionResponse struct {
Version string `json:"version"`
}
func compareJSON(a, b []byte) (bool, error) {
var j1, j2 interface{}
if err := json.Unmarshal(a, &j1); err != nil {
@ -156,12 +143,10 @@ func compareJSON(a, b []byte) (bool, error) {
func BenchmarkDiagnostics(b *testing.B) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
// Create a new client.
d := diagnostics.New(server.URL)
d := NewDiagnosticsCollector(server.URL)
d.SetLogger(ioutil.Discard)
defer d.Close()
prev := runtime.GOMAXPROCS(4)
defer runtime.GOMAXPROCS(prev)

66
docs/administration.md
View file

@ -5,6 +5,7 @@ nav = [
"Installing in production",
"Importing and Exporting Data",
"Versioning",
"Resizing the Cluster",
"Backup/restore",
]
+++
@ -93,6 +94,71 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ
When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes.
### Resizing the Cluster
If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process.
#### Adding a Node
You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries.
If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately.
#### Removing a Node
In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself.
To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing an `/id` request to the node:
``` request
curl localhost:10102/id
```
``` response
40a891fa-243b-4d71-ae24-4f5c78a0f4b1
```
If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node:
``` request
curl localhost:10101/status
```
``` response
{
"state":"NORMAL",
"nodes":[
{"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
{"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
{"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
]
}
```
Once you have the ID of the node that you want to remove from the cluster, issue the following request:
```
curl localhost:10101/cluster/resize/remove-node \
-X POST \
-d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}'
```
At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries.
Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator).
#### Aborting a Resize Job
If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node.
For example, if your coordinator node is `localhost:10101`, then you can run:
```
curl localhost:10101/cluster/resize/abort -X POST
```
This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began.
#### Changing the Coordinator
In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator.
```
curl localhost:10101/cluster/resize/set-coordinator \
-X POST \
-d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}'
```
### Backup/restore
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster.

75
docs/configuration.md
View file

@ -27,24 +27,18 @@ Every command line flag has a corresponding environment variable. The environmen
### Config file
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file:
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file:
```toml
[cluster]
poll-interval = "2m0s"
coordinator = true
replicas = 1
```
Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes:
```toml
[cluster]
hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"]
```
### All Options
#### Anti Entropy Interval
* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync.
* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
* Flag: `--anti-entropy.interval="10m0s"`
* Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"`
* Config:
@ -84,7 +78,7 @@ Any flag that has a value that is a comma separated list on the command line bec
* Config:
```toml
log_path = "/path/to/logfile"
log-path = "/path/to/logfile"
```
#### Max Writes Per Request
@ -110,16 +104,16 @@ Any flag that has a value that is a comma separated list on the command line bec
port = 11101
```
#### Gossip Seed
#### Gossip Seeds
* Description: When using the gossip [Cluster Type](#cluster-type), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`.
* Flag: `--gossip.seed="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEED="localhost:11101"`
* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms.
* Flag: `--gossip.seeds="localhost:11101"`
* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101"`
* Config:
```toml
[gossip]
seed = "localhost:11101"
seeds = ["localhost:11101"]
```
#### Gossip Key
@ -133,28 +127,16 @@ Any flag that has a value that is a comma separated list on the command line bec
key = "/var/secret/gossip.key32"
```
#### Cluster Hosts
#### Cluster Coordinator
* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms.
* Flag: `--cluster.hosts="localhost:10101"`
* Env: `PILOSA_CLUSTER_HOSTS="localhost:10101"`
* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
* Flag: `cluster.coordinator`
* Env: `PILOSA_CLUSTER_COORDINATOR`
* Config:
```toml
[cluster]
hosts = ["localhost:10101"]
```
#### Cluster Poll Interval
* Description: Polling interval for cluster.
* Flag: `cluster.poll-interval="1m0s"`
* Env: `PILOSA_CLUSTER_POLL_INTERVAL="1m0s"`
* Config:
```toml
[cluster]
poll-interval = "1m0s"
coordinator = true
```
#### Cluster Long Query Time
@ -218,7 +200,7 @@ Any flag that has a value that is a comma separated list on the command line bec
[profile]
cpu-time = "30s"
```
#### Metric Service
* Description: Which stats service to use. Choose from [statsd, expvar].
* Flag: `--metric.service=statsd`
@ -317,8 +299,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = true
#### Node 1
@ -331,8 +312,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
#### Node 2
@ -345,8 +325,7 @@ A three node cluster running on different hosts could be minimally configured as
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
coordinator = false
### Example Cluster Configuration (HTTPS)
@ -365,8 +344,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -384,8 +362,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -403,8 +380,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows.
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://node0.pilosa.com:10101","https://node1.pilosa.com:10101","https://node2.pilosa.com:10101"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -426,8 +402,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -445,8 +420,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
@ -464,8 +438,7 @@ You can run a cluster on the same host using the configuration above with a few
[cluster]
replicas = 1
type = "gossip"
hosts = ["https://localhost:10100","https://localhost:10101","https://localhost:10102"]
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"

8
docs/getting-started.md
View file

@ -35,7 +35,7 @@ Let's make sure Pilosa is running:
curl localhost:10101/status
```
``` response
{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}
{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]}
```
### Sample Project
@ -46,7 +46,9 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term
#### Create the Schema
The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
Note:
The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
``` request
curl localhost:10101/schema
```
@ -107,7 +109,7 @@ docker cp language.csv pilosa:/language.csv
docker exec -it pilosa /pilosa import -i repository -f language /language.csv
```
Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `languages.txt` to see the mapping for languages.
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
### Input Definition

55
event.go Normal file
View file

@ -0,0 +1,55 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// NodeEventType are the types of events that can be sent from the
// ChannelEventDelegate.
type NodeEventType int
const (
NodeJoin NodeEventType = iota
NodeLeave
NodeUpdate
)
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
}
// EventHandler is the interface for the pilosa object which knows how to
// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
type EventHandler interface {
ReceiveEvent(e *NodeEvent) error
}
// EventReceiver is the interface for the object which will listen for and
// decode broadcast messages before passing them to pilosa to handle. The
// implementation of this could be an http server which listens for messages,
// gets the protobuf payload, and then passes it to
// EventHandler.ReceiveMessage.
type EventReceiver interface {
// Start starts listening for broadcast messages - it should return
// immediately, spawning a goroutine if necessary.
Start(EventHandler) error
}
type nopEventReceiver struct{}
func (n *nopEventReceiver) Start(e EventHandler) error { return nil }
// NopEventReceiver is a no-op implementation of the EventReceiver.
var NopEventReceiver = &nopEventReceiver{}

27
executor.go
View file

@ -40,8 +40,7 @@ type Executor struct {
Holder *Holder
// Local hostname & cluster configuration.
Scheme string
Host string
Node *Node
Cluster *Cluster
// Client used for remote requests.
@ -957,7 +956,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
ret := false
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
if err != nil {
return false, err
@ -1062,7 +1061,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
if err != nil {
return false, err
@ -1141,7 +1140,7 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1199,7 +1198,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1286,7 +1285,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1345,7 +1344,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterHost(e.Host)
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
@ -1372,12 +1371,8 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
Slices: slices,
Remote: true,
}
uri, err := NewURIFromAddress(node.Host)
if err != nil {
return nil, err
}
uri.SetScheme(node.Scheme)
ctx = context.WithValue(ctx, "uri", uri)
ctx = context.WithValue(ctx, "uri", node.URI)
pb, err := e.client.ExecuteQuery(ctx, index, pbreq)
if err != nil {
return nil, err
@ -1456,7 +1451,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.Nodes).Clone()
} else {
nodes = []*Node{e.Cluster.NodeByHost(e.Host)}
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
@ -1512,7 +1507,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
resp := mapResponse{node: n, slices: nodeSlices}
// Send local slices to mapper, otherwise remote exec.
if n.Host == e.Host {
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)

40
executor_test.go
View file

@ -897,8 +897,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Scheme = "http"
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -941,7 +945,13 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -971,7 +981,13 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
@ -1017,7 +1033,13 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
@ -1064,7 +1086,13 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int

42
filesystem.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"net/http"
)
// Ensure nopFileSystem implements interface.
var _ FileSystem = &nopFileSystem{}
// FileSystem represents an interface for a WebUI file system.
type FileSystem interface {
New() (http.FileSystem, error)
}
func init() {
NopFileSystem = &nopFileSystem{}
}
// NopFileSystem represents a FileSystem that returns an error if called.
var NopFileSystem FileSystem
type nopFileSystem struct{}
// New is a no-op implementation of FileSystem New method.
func (n *nopFileSystem) New() (http.FileSystem, error) {
return nil, fmt.Errorf("file system not implemented")
}

23
fragment.go
View file

@ -108,7 +108,7 @@ type Fragment struct {
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
RowAttrStore *AttrStore
RowAttrStore AttrStore
stats StatsClient
}
@ -256,6 +256,7 @@ func (f *Fragment) openCache() error {
f.cache = NewLRUCache(f.CacheSize)
case CacheTypeNone:
f.cache = NewNopCache()
return nil
default:
return ErrInvalidCacheType
}
@ -1455,6 +1456,10 @@ func (f *Fragment) flushCache() error {
return nil
}
if f.CacheType == CacheTypeNone {
return nil
}
// Retrieve a list of row ids from the cache.
ids := f.cache.IDs()
@ -1681,7 +1686,7 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Host string
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -1711,17 +1716,14 @@ func (s *FragmentSyncer) SyncFragment() error {
blockSets := make([][]FragmentBlock, 0, len(nodes))
for _, node := range nodes {
// Read local blocks.
if node.Host == s.Host {
if node.ID == s.Node.ID {
b := s.Fragment.Blocks()
blockSets = append(blockSets, b)
continue
}
// Retrieve remote blocks.
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return err
@ -1788,7 +1790,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
var pairSets []PairSet
var clients []InternalClient
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
if s.Host == node.Host {
if s.Node.ID == node.ID {
continue
}
@ -1797,10 +1799,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
return nil
}
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
clients = append(clients, client)
// Only sync the standard block.

2
fragment_test.go
View file

@ -702,7 +702,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
Fragment: frag,
RowAttrStore: test.MustOpenAttrStore(),
}
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
panic(err)
}

310
frame.go
View file

@ -43,29 +43,27 @@ const (
// Frame represents a container for views.
type Frame struct {
mu sync.RWMutex
path string
index string
name string
timeQuantum TimeQuantum
schema *FrameSchema
mu sync.RWMutex
path string
index string
name string
views map[string]*View
// Row attribute storage and cache
rowAttrStore *AttrStore
rowAttrStore AttrStore
broadcaster Broadcaster
Stats StatsClient
// Frame settings.
// Frame options.
rowLabel string
cacheType string
inverseEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
rangeEnabled bool
// Cache size for ranked frames
cacheSize uint32
fields []*Field
LogOutput io.Writer
}
@ -78,22 +76,24 @@ func NewFrame(path, index, name string) (*Frame, error) {
}
return &Frame{
path: path,
index: index,
name: name,
schema: &FrameSchema{},
path: path,
index: index,
name: name,
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
views: make(map[string]*View),
rowAttrStore: NopAttrStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
rowLabel: DefaultRowLabel,
inverseEnabled: DefaultInverseEnabled,
rangeEnabled: DefaultRangeEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
rangeEnabled: DefaultRangeEnabled,
//fields
LogOutput: ioutil.Discard,
}, nil
@ -109,7 +109,7 @@ func (f *Frame) Index() string { return f.index }
func (f *Frame) Path() string { return f.path }
// RowAttrStore returns the attribute storage.
func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
@ -230,7 +230,7 @@ func (f *Frame) options() FrameOptions {
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.schema.Fields,
Fields: f.fields,
}
}
@ -244,8 +244,6 @@ func (f *Frame) Open() error {
if err := f.loadMeta(); err != nil {
return err
} else if err := f.loadSchema(); err != nil {
return err
}
if err := f.openViews(); err != nil {
@ -304,12 +302,13 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.timeQuantum = ""
f.rowLabel = DefaultRowLabel
f.cacheType = DefaultCacheType
f.inverseEnabled = DefaultInverseEnabled
f.rangeEnabled = DefaultRangeEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
f.rangeEnabled = DefaultRangeEnabled
//f.fields
return nil
} else if err != nil {
return err
@ -320,17 +319,16 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rowLabel = pb.RowLabel
f.inverseEnabled = pb.InverseEnabled
f.rangeEnabled = pb.RangeEnabled
f.cacheSize = pb.CacheSize
// Copy cache type.
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
}
f.cacheSize = pb.CacheSize
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rangeEnabled = pb.RangeEnabled
f.fields = decodeFields(pb.Fields)
return nil
}
@ -352,35 +350,6 @@ func (f *Frame) saveMeta() error {
return nil
}
// loadSchema reads the schema for the frame.
func (f *Frame) loadSchema() error {
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".schema"))
if os.IsNotExist(err) {
f.schema = &FrameSchema{}
return nil
} else if err != nil {
return err
}
var pb internal.FrameSchema
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
f.schema = decodeFrameSchema(&pb)
return nil
}
// saveSchema writes the current schema to disk.
func (f *Frame) saveSchema() error {
if buf, err := proto.Marshal(encodeFrameSchema(f.schema)); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(f.path, ".schema"), buf, 0666); err != nil {
return err
}
return nil
}
// Close closes the frame and its views.
func (f *Frame) Close() error {
f.mu.Lock()
@ -402,16 +371,11 @@ func (f *Frame) Close() error {
return nil
}
// Schema returns the frame's current schema.
func (f *Frame) Schema() *FrameSchema {
// Field returns a field by name.
func (f *Frame) Field(name string) *Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.schema
}
// Field returns a field from the schema by name.
func (f *Frame) Field(name string) *Field {
for _, field := range f.Schema().Fields {
for _, field := range f.fields {
if field.Name == name {
return field
}
@ -419,7 +383,24 @@ func (f *Frame) Field(name string) *Field {
return nil
}
// CreateField creates a new field on the schema.
// Fields returns the fields on the frame.
func (f *Frame) Fields() []*Field {
f.mu.RLock()
defer f.mu.RUnlock()
return f.fields
}
// HasField returns true if a field exists on the frame.
func (f *Frame) HasField(name string) bool {
for _, fld := range f.fields {
if fld.Name == name {
return true
}
}
return false
}
// CreateField creates a new field on the frame.
func (f *Frame) CreateField(field *Field) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -429,18 +410,35 @@ func (f *Frame) CreateField(field *Field) error {
return ErrFrameFieldsNotAllowed
}
// Copy schema and append field.
schema := f.schema.Clone()
if err := schema.AddField(field); err != nil {
// Append field.
if err := f.addField(field); err != nil {
return err
}
f.schema = schema
f.saveSchema()
f.saveMeta()
return nil
}
// addField adds a single field to fields.
func (f *Frame) addField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if f.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
f.fields = append(f.fields, field)
// Sort fields by name.
sort.Slice(f.fields, func(i, j int) bool {
return f.fields[i].Name < f.fields[j].Name
})
return nil
}
// GetFields returns a list of all the fields in the frame.
func (f *Frame) GetFields() (*FrameSchema, error) {
func (f *Frame) GetFields() ([]*Field, error) {
f.mu.RLock()
defer f.mu.RUnlock()
@ -449,12 +447,12 @@ func (f *Frame) GetFields() (*FrameSchema, error) {
return nil, ErrFrameFieldsNotAllowed
}
err := f.loadSchema()
err := f.loadMeta()
if err != nil {
return nil, err
}
return f.schema, nil
return f.fields, nil
}
// DeleteField deletes an existing field on the schema.
@ -467,12 +465,10 @@ func (f *Frame) DeleteField(name string) error {
return ErrFrameFieldsNotAllowed
}
// Copy schema and remove field.
schema := f.schema.Clone()
if err := schema.DeleteField(name); err != nil {
// Remove field.
if err := f.deleteField(name); err != nil {
return err
}
f.schema = schema
// Remove views.
viewName := ViewFieldPrefix + name
@ -489,6 +485,18 @@ func (f *Frame) DeleteField(name string) error {
return nil
}
// deleteField removes a single field from fields.
func (f *Frame) deleteField(name string) error {
for i, field := range f.fields {
if field.Name == name {
copy(f.fields[i:], f.fields[i+1:])
f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
f.mu.Lock()
@ -543,6 +551,18 @@ func (f *Frame) Views() []*View {
return other
}
// viewNames returns a list of all views (as a string) in the frame.
func (f *Frame) viewNames() []string {
f.mu.Lock()
defer f.mu.Unlock()
other := make([]string, 0, len(f.views))
for viewName, _ := range f.views {
other = append(other, viewName)
}
return other
}
// RecalculateCaches recalculates caches on every view in the frame.
func (f *Frame) RecalculateCaches() {
for _, view := range f.Views() {
@ -551,27 +571,54 @@ func (f *Frame) RecalculateCaches() {
}
// CreateViewIfNotExists returns the named view, creating it if necessary.
// Additionally, a CreateViewMessage is sent to the cluster.
func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
view, created, err := f.createViewIfNotExistsBase(name)
if err != nil {
return nil, err
}
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&internal.CreateViewMessage{
Index: f.index,
Frame: f.name,
View: name,
})
if err != nil {
return nil, err
}
}
return view, nil
}
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, ErrFrameInverseDisabled
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
if view := f.views[name]; view != nil {
return view, nil
return view, false, nil
}
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return nil, err
return nil, false, err
}
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
return view, nil
return view, true, nil
}
func (f *Frame) newView(path, name string) *View {
@ -956,8 +1003,9 @@ func encodeFrames(a []*Frame) []*internal.Frame {
func encodeFrame(f *Frame) *internal.Frame {
fo := f.options()
return &internal.Frame{
Name: f.name,
Meta: fo.Encode(),
Name: f.name,
Meta: fo.Encode(),
Views: f.viewNames(),
}
}
@ -969,8 +1017,9 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FrameInfo represents schema information for a frame.
type FrameInfo struct {
Name string `json:"name"`
Views []*ViewInfo `json:"views,omitempty"`
Name string `json:"name"`
Options FrameOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type frameInfoSlice []*FrameInfo
@ -992,6 +1041,13 @@ type FrameOptions struct {
// Encode converts o into its internal representation.
func (o *FrameOptions) Encode() *internal.FrameMeta {
return encodeFrameOptions(o)
}
func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
if o == nil {
return nil
}
return &internal.FrameMeta{
RowLabel: o.RowLabel,
InverseEnabled: o.InverseEnabled,
@ -1003,74 +1059,18 @@ func (o *FrameOptions) Encode() *internal.FrameMeta {
}
}
// FrameSchema represents the list of fields on a frame.
type FrameSchema struct {
Fields []*Field
}
// Clone returns a copy of s.
func (s *FrameSchema) Clone() *FrameSchema {
other := &FrameSchema{Fields: make([]*Field, len(s.Fields))}
copy(other.Fields, s.Fields)
return other
}
// HasField returns true if a field exists on the schema.
func (s *FrameSchema) HasField(name string) bool {
for _, f := range s.Fields {
if f.Name == name {
return true
}
}
return false
}
// AddField adds a single field to the schema.
func (s *FrameSchema) AddField(field *Field) error {
if err := ValidateField(field); err != nil {
return err
} else if s.HasField(field.Name) {
return ErrFieldExists
}
// Add field to list.
s.Fields = append(s.Fields, field)
// Sort fields by name.
sort.Slice(s.Fields, func(i, j int) bool {
return s.Fields[i].Name < s.Fields[j].Name
})
return nil
}
// DeleteField removes a single field from the schema.
func (s *FrameSchema) DeleteField(name string) error {
for i, field := range s.Fields {
if field.Name == name {
copy(s.Fields[i:], s.Fields[i+1:])
s.Fields, s.Fields[len(s.Fields)-1] = s.Fields[:len(s.Fields)-1], nil
return nil
}
}
return ErrFieldNotFound
}
func encodeFrameSchema(schema *FrameSchema) *internal.FrameSchema {
if schema == nil {
func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
if options == nil {
return nil
}
return &internal.FrameSchema{
Fields: encodeFields(schema.Fields),
}
}
func decodeFrameSchema(schema *internal.FrameSchema) *FrameSchema {
if schema == nil {
return nil
}
return &FrameSchema{
Fields: decodeFields(schema.Fields),
return &FrameOptions{
RowLabel: options.RowLabel,
InverseEnabled: options.InverseEnabled,
RangeEnabled: options.RangeEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

41
gc.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// Ensure nopGCNotifier implements interface.
var _ GCNotifier = &nopGCNotifier{}
// GCNotifier represents an interface for garbage collection notificationss.
type GCNotifier interface {
Close()
AfterGC() <-chan struct{}
}
func init() {
NopGCNotifier = &nopGCNotifier{}
}
// NopGCNotifier represents a GCNotifier that doesn't do anything.
var NopGCNotifier GCNotifier
type nopGCNotifier struct{}
// Close is a no-op implementation of GCNotifier Close method.
func (n *nopGCNotifier) Close() {}
// AfterGC is a no-op implementation of GCNotifier AfterGC method.
func (c *nopGCNotifier) AfterGC() <-chan struct{} {
return nil
}

44
gcnotify/gcnotify.go Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gcnotify
import (
"github.com/CAFxX/gcnotifier"
"github.com/pilosa/pilosa"
)
// Ensure ActiveGCNotifier implements interface.
var _ pilosa.GCNotifier = &ActiveGCNotifier{}
type ActiveGCNotifier struct {
gcn *gcnotifier.GCNotifier
}
// NewActiveGCNotifier creates an active GCNotifier.
func NewActiveGCNotifier() *ActiveGCNotifier {
return &ActiveGCNotifier{
gcn: gcnotifier.New(),
}
}
// Close implements the GCNotifier interface.
func (n *ActiveGCNotifier) Close() {
n.gcn.Close()
}
// AfterGC implements the GCNotifier interface.
func (n *ActiveGCNotifier) AfterGC() <-chan struct{} {
return n.gcn.AfterGC()
}

115
gopsutil/systeminfo.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil
import (
"github.com/pilosa/pilosa"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
)
var _ pilosa.SystemInfo = NewSystemInfo()
// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS.
type SystemInfo struct {
platform string
family string
osVersion string
}
// Uptime returns the system uptime in seconds.
func (s *SystemInfo) Uptime() (uptime uint64, err error) {
hostInfo, err := host.Info()
if err != nil {
return 0, err
}
return hostInfo.Uptime, nil
}
// collectPlatformInfo fetches and caches system platform information.
func (s *SystemInfo) collectPlatformInfo() error {
var err error
if s.platform == "" {
s.platform, s.family, s.osVersion, err = host.PlatformInformation()
if err != nil {
return err
}
}
return nil
}
// Platform returns the system platform.
func (s *SystemInfo) Platform() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.platform, nil
}
// Family returns the system family.
func (s *SystemInfo) Family() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.family, err
}
// OSVersion returns the OS Version.
func (s *SystemInfo) OSVersion() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
}
return s.osVersion, err
}
// MemFree returns the amount of free memory in bytes.
func (s *SystemInfo) MemFree() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Free, err
}
// MemTotal returns the amount of total memory in bytes.
func (s *SystemInfo) MemTotal() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Total, err
}
// MemUsed returns the amount of used memory in bytes.
func (s *SystemInfo) MemUsed() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
}
return memInfo.Used, err
}
// KernelVersion returns the kernel version as a string.
func (s *SystemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *SystemInfo {
return &SystemInfo{}
}

77
gopsutil/systeminfo_test.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gopsutil_test
import (
"log"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gopsutil"
)
func TestSystemInfo(t *testing.T) {
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
// Uptime()(uint64, error)
// Platform()(string, error)
// Family()(string, error)
// OSVersion()(string, error)
// KernelVersion()(string, error)
// MemFree()(uint64, error)
// MemTotal()(uint64, error)
// MemUsed()(uint64, error)
//
uptime, err := systemInfo.Uptime()
if err != nil || uptime == 0 {
t.Fatalf("Error collecting uptime (error: %v)", err)
}
platform, err := systemInfo.Platform()
if err != nil {
t.Fatalf("Error getting platform. (platform: %v, error: %v)", platform, err)
}
family, err := systemInfo.Family()
if err != nil {
t.Fatalf("Error getting OS family. (family: %v, error: %v)", family, err)
}
osversion, err := systemInfo.OSVersion()
if err != nil {
t.Fatalf("Error getting OS version. (osversion: %v, error: %v)", osversion, err)
}
kernelversion, err := systemInfo.KernelVersion()
if err != nil {
t.Fatalf("Error getting kernel version. (kernelversion: %v, error: %v)", kernelversion, err)
}
memfree, err := systemInfo.MemFree()
if err != nil {
t.Fatalf("Error getting memfree. (memfree: %v, error: %v)", memfree, err)
}
memused, err := systemInfo.MemUsed()
if err != nil {
t.Fatalf("Error getting memused. (memused: %v, error: %v)", memused, err)
}
memtotal, err := systemInfo.MemTotal()
log.Println(memtotal)
if err != nil {
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
}
}

542
gossip/gossip.go
View file

@ -17,11 +17,16 @@ package gossip
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
@ -29,15 +34,15 @@ import (
"github.com/pkg/errors"
)
// Ensure GossipNodeSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipNodeSet{}
var _ pilosa.Gossiper = &GossipNodeSet{}
var _ memberlist.Delegate = &GossipNodeSet{}
// Ensure GossipMemberSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipMemberSet{}
var _ pilosa.Gossiper = &GossipMemberSet{}
var _ memberlist.Delegate = &GossipMemberSet{}
// GossipNodeSet represents a gossip implementation of NodeSet using memberlist
// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster
// GossipNodeSet also represents an implementation of memberlist.Delegate
type GossipNodeSet struct {
// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
type GossipMemberSet struct {
mu sync.RWMutex
node *pilosa.Node
memberlist *memberlist.Memberlist
handler pilosa.BroadcastHandler
@ -50,46 +55,60 @@ type GossipNodeSet struct {
LogOutput io.Writer
}
// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
func (g *GossipNodeSet) Nodes() []*pilosa.Node {
a := make([]*pilosa.Node, 0, g.memberlist.NumMembers())
for _, n := range g.memberlist.Members() {
a = append(a, &pilosa.Node{Scheme: "gossip", Host: n.Name})
}
return a
}
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler
func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error {
// Start implements the BroadcastReceiver interface and sets the BroadcastHandler.
func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error {
g.handler = h
return nil
}
// Seed returns the gossipSeed determined by the config.
func (g *GossipNodeSet) Seed() string {
return g.config.gossipSeed
// GetBindAddr returns the gossip bind address based on config and auto bind port.
// This method is currently only used in a test scenario where a second node needs
// the auto-bind address of the first node to use as its gossip seed.
func (g *GossipMemberSet) GetBindAddr() string {
return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort)
}
// Open implements the NodeSet interface to start network activity.
func (g *GossipNodeSet) Open() error {
// Open implements the MemberSet interface to start network activity.
func (g *GossipMemberSet) Open(n *pilosa.Node) error {
if g.handler == nil {
return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()")
return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()")
}
ml, err := memberlist.Create(g.config.memberlistConfig)
g.node = n
err := error(nil)
g.mu.Lock()
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
if err != nil {
return errors.Wrap(err, "creating memberlist")
}
g.memberlist = ml
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
g.mu.RLock()
defer g.mu.RUnlock()
return g.memberlist.NumMembers()
},
RetransmitMult: 3,
}
// attach to gossip seed node
nodes := []*pilosa.Node{&pilosa.Node{Scheme: "gossip", Host: g.config.gossipSeed}} //TODO: support a list of seeds
err = g.joinWithRetry(pilosa.Nodes(nodes).Hosts())
var uris = make([]*pilosa.URI, len(g.config.gossipSeeds))
for i, addr := range g.config.gossipSeeds {
uris[i], err = pilosa.NewURIFromAddress(addr)
if err != nil {
return fmt.Errorf("new uri from address: %s", err)
}
}
var nodes = make([]*pilosa.Node, len(uris))
for i, uri := range uris {
nodes[i] = &pilosa.Node{URI: *uri}
}
g.mu.RLock()
err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings())
g.mu.RUnlock()
if err != nil {
return errors.Wrap(err, "joinWithRetry")
}
@ -97,7 +116,7 @@ func (g *GossipNodeSet) Open() error {
}
// joinWithRetry wraps the standard memberlist Join function in a retry.
func (g *GossipNodeSet) joinWithRetry(hosts []string) error {
func (g *GossipMemberSet) joinWithRetry(hosts []string) error {
err := retry(60, 2*time.Second, func() error {
_, err := g.memberlist.Join(hosts)
return err
@ -121,21 +140,334 @@ func retry(attempts int, sleep time.Duration, fn func() error) (err error) {
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
// logger returns a logger for the GossipNodeSet.
func (g *GossipNodeSet) logger() *log.Logger {
// logger returns a logger for the GossipMemberSet.
func (g *GossipMemberSet) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
////////////////////////////////////////////////////////////////
type gossipConfig struct {
gossipSeed string
gossipSeeds []string
memberlistConfig *memberlist.Config
}
// NewGossipMemberSetWithTransport returns a new instance of GossipMemberSet given a Transport.
func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport *Transport, server *pilosa.Server) (*GossipMemberSet, error) {
g := &GossipMemberSet{
LogOutput: server.LogOutput,
}
port := transport.Net.GetAutoBindPort()
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err)
}
host := bindURI.Host()
var gossipKey []byte
if cfg.Gossip.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key)
if err != nil {
return nil, fmt.Errorf("reading gossip key: %s", err)
}
}
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = transport.Net
conf.Name = name
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout)
conf.SuspicionMult = cfg.Gossip.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval)
conf.GossipNodes = cfg.Gossip.Nodes
conf.GossipInterval = time.Duration(cfg.Gossip.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime)
//
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeeds: cfg.Gossip.Seeds,
}
g.statusHandler = server
return g, nil
}
// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) {
port, err := strconv.Atoi(cfg.Gossip.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
bindURI, err := pilosa.NewURIFromAddress(cfg.Bind)
if err != nil {
return nil, fmt.Errorf("getting uri from bind address: %s", err)
}
host := bindURI.Host()
// Set up the transport.
transport, err := NewTransport(host, port)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
return NewGossipMemberSetWithTransport(name, cfg, transport, server)
}
// SendSync implementation of the Broadcaster interface.
func (g *GossipMemberSet) SendSync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
mlist := g.memberlist
// Direct sends the message directly to every node.
// An error from any node raises an error on the entire operation.
//
// Gossip uses the gossip protocol to eventually deliver the message
// to every node.
var eg errgroup.Group
for _, n := range mlist.Members() {
// Don't send the message to the local node.
if n == mlist.LocalNode() {
continue
}
node := n
eg.Go(func() error {
return mlist.SendToTCP(node, msg)
})
}
return eg.Wait()
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
b := &broadcast{
msg: msg,
notify: nil,
}
g.broadcasts.QueueBroadcast(b)
return nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
if err != nil {
g.logger().Printf("marshal message error: %s", err)
return []byte{}
}
return buf
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipMemberSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
return
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side's LocalState.
func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
}
}
// GossipEventReceiver is used to enable an application to receive
// events about joins and leaves over a channel.
//
// Care must be taken that events are processed in a timely manner from
// the channel, since this delegate will block until an event can be sent.
type GossipEventReceiver struct {
ch chan memberlist.NodeEvent
eventHandler pilosa.EventHandler
// The writer for any logging.
LogOutput io.Writer
}
// NewGossipEventReceiver returns a new instance of GossipEventReceiver.
func NewGossipEventReceiver(logOutput io.Writer) *GossipEventReceiver {
return &GossipEventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
LogOutput: logOutput,
}
}
func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n}
}
func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n}
}
func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n}
}
// Start implements the pilosa.EventReceiver interface and sets the EventHandler.
func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error {
g.eventHandler = h
go g.listen()
return nil
}
// logger returns a logger for the GossipEventReceiver.
func (g *GossipEventReceiver) logger() *log.Logger {
return log.New(g.LogOutput, "", log.LstdFlags)
}
func (g *GossipEventReceiver) listen() {
var nodeEventType pilosa.NodeEventType
for {
e := <-g.ch
switch e.Event {
case memberlist.NodeJoin:
nodeEventType = pilosa.NodeJoin
case memberlist.NodeLeave:
nodeEventType = pilosa.NodeLeave
case memberlist.NodeUpdate:
nodeEventType = pilosa.NodeUpdate
default:
continue
}
// Get the node from the event.Node meta data.
var n internal.Node
if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta data")
}
node := pilosa.DecodeNode(&n)
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: node,
}
if err := g.eventHandler.ReceiveEvent(ne); err != nil {
g.logger().Printf("receive event error: %s", err)
}
}
}
// broadcast represents an implementation of memberlist.Broadcast
type broadcast struct {
msg []byte
notify chan<- struct{}
}
func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
return false
}
func (b *broadcast) Message() []byte {
return b.msg
}
func (b *broadcast) Finished() {
if b.notify != nil {
close(b.notify)
}
}
// Transport is a gossip transport for binding to a port.
type Transport struct {
//memberlist.Transport
Net *memberlist.NetTransport
URI *pilosa.URI
}
// NewTransport returns a NetTransport based on the given host and port.
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifiying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
net, err := newTransport(conf)
if err != nil {
return nil, fmt.Errorf("new transport: %s", err)
}
uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort()))
if err != nil {
return nil, fmt.Errorf("new uri from host port: %s", err)
}
return &Transport{
Net: net,
URI: uri,
}, nil
}
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
// This is useful for test cases where specifying a port is not reasonable.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if conf.LogOutput != nil && conf.Logger != nil {
return nil, fmt.Errorf("Cannot specify both LogOutput and Logger. Please choose a single log configuration setting.")
@ -189,144 +521,6 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
if err != nil {
return nil, fmt.Errorf("Could not set up network transport: %v", err)
}
if conf.BindPort == 0 {
port := nt.GetAutoBindPort()
conf.BindPort = port
conf.AdvertisePort = port
logger.Printf("[DEBUG] Using dynamic bind port %d", port)
}
return nt, nil
}
// NewGossipNodeSet returns a new instance of GossipNodeSet.
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server, secretKey []byte) (*GossipNodeSet, error) {
g := &GossipNodeSet{
LogOutput: server.LogOutput,
}
conf := memberlist.DefaultWANConfig()
conf.BindPort = gossipPort
conf.AdvertisePort = gossipPort
//TODO: pull memberlist config from pilosa.cfg file
g.config = &gossipConfig{
memberlistConfig: conf,
gossipSeed: gossipSeed,
}
g.config.memberlistConfig.Name = name
g.config.memberlistConfig.BindAddr = gossipHost
g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost)
g.config.memberlistConfig.Delegate = g
g.config.memberlistConfig.SecretKey = secretKey
g.statusHandler = server
// set up the transport
transport, err := newTransport(g.config.memberlistConfig)
if err != nil {
return nil, err
}
g.config.memberlistConfig.Transport = transport
// If no gossipSeed is provided, use local host:port.
if gossipSeed == "" {
g.config.gossipSeed = fmt.Sprintf("%s:%d", gossipHost, g.config.memberlistConfig.BindPort)
}
return g, nil
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipNodeSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return err
}
b := &broadcast{
msg: msg,
notify: nil,
}
g.broadcasts.QueueBroadcast(b)
return nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipNodeSet) NodeMeta(limit int) []byte {
return []byte{}
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipNodeSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
if err != nil {
g.logger().Printf("unmarshal message error: %s", err)
return
}
if err := g.handler.ReceiveMessage(m); err != nil {
g.logger().Printf("receive message error: %s", err)
return
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipNodeSet) LocalState(join bool) []byte {
pb, err := g.statusHandler.LocalStatus()
if err != nil {
g.logger().Printf("error getting local state, err=%s", err)
return []byte{}
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
if err != nil {
g.logger().Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side side's LocalState.
func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.statusHandler.HandleRemoteStatus(&pb)
if err != nil {
g.logger().Printf("merge state error: %s", err)
}
}
// broadcast represents an implementation of memberlist.Broadcast
type broadcast struct {
msg []byte
notify chan<- struct{}
}
func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
return false
}
func (b *broadcast) Message() []byte {
return b.msg
}
func (b *broadcast) Finished() {
if b.notify != nil {
close(b.notify)
}
}

298
handler.go
View file

@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:generate statik -src=./webui
package pilosa
import (
@ -44,10 +42,6 @@ import (
"github.com/pilosa/pilosa/pql"
"unicode"
// Allow building Pilosa without the web UI.
_ "github.com/pilosa/pilosa/statik"
"github.com/rakyll/statik/fs"
)
// Handler represents an HTTP handler.
@ -57,12 +51,16 @@ type Handler struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
FileSystem FileSystem
// Local hostname & cluster configuration.
URI *URI
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Router *mux.Router
Router *mux.Router
NormalRouter *mux.Router
RestrictedRouter *mux.Router
// The execution engine for running queries.
Executor interface {
@ -96,13 +94,36 @@ type errorResponse struct {
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler() *Handler {
handler := &Handler{
Broadcaster: NopBroadcaster,
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
FileSystem: NopFileSystem,
LogOutput: os.Stderr,
}
handler.Router = NewRouter(handler)
BuildRouters(handler)
handler.populateValidators()
return handler
}
// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints.
func BuildRouters(handler *Handler) {
router := mux.NewRouter()
loadCommon(router, handler)
loadNormal(router, handler)
handler.NormalRouter = router
router.Use(handler.queryArgValidator)
// Restricted router.
router = mux.NewRouter()
loadCommon(router, handler)
loadRestricted(router, handler)
handler.RestrictedRouter = router
router.Use(handler.queryArgValidator)
handler.SetRestricted()
}
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index")
@ -135,17 +156,46 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
})
}
// NewRouter creates a Gorilla Mux http router.
func NewRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
// SetNormal is a method of the SecurityManager interface which provides normal URI routing.
func (h *Handler) SetNormal() {
h.Router = h.NormalRouter
}
// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing.
func (h *Handler) SetRestricted() {
h.Router = h.RestrictedRouter
}
func loadCommon(router *mux.Router, handler *Handler) {
router.HandleFunc("/", handler.handleWebUI).Methods("GET")
router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET")
router.HandleFunc("/id", handler.handleGetID).Methods("GET")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.Use(handler.queryArgValidator)
}
func loadRestricted(router *mux.Router, handler *Handler) {
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted)
router.Use(handler.queryArgValidator)
}
func loadNormal(router *mux.Router, handler *Handler) {
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData")
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
@ -172,14 +222,7 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH")
router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET").Name("GetSliceMax")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/id", handler.handleGetID).Methods("GET")
// TODO: Apply MethodNotAllowed statuses to all endpoints.
// Ideally this would be automatic, as described in this (wontfix) ticket:
@ -187,9 +230,10 @@ func NewRouter(handler *Handler) *mux.Router {
// For now we just do it for the most commonly used handler, /query
router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
router.Use(handler.queryArgValidator)
}
return router
func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) {
http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.Cluster.State()), http.StatusMethodNotAllowed)
}
func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) {
@ -242,13 +286,13 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound)
return
}
statikFS, err := fs.New()
filesystem, err := h.FileSystem.New()
if err != nil {
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
return
}
http.FileServer(statikFS).ServeHTTP(w, r)
http.FileServer(filesystem).ServeHTTP(w, r)
}
// handleGetSchema handles GET /schema requests.
@ -262,13 +306,16 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
// handleGetStatus handles GET /status requests.
func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
status, err := h.StatusHandler.ClusterStatus()
pb, err := h.StatusHandler.ClusterStatus()
if err != nil {
h.logger().Printf("cluster status error: %s", err)
return
}
cs := pb.(*internal.ClusterStatus)
if err := json.NewEncoder(w).Encode(getStatusResponse{
Status: status,
State: cs.State,
Nodes: DecodeNodes(cs.Nodes),
}); err != nil {
h.logger().Printf("write status response error: %s", err)
}
@ -279,7 +326,8 @@ type getSchemaResponse struct {
}
type getStatusResponse struct {
Status proto.Message `json:"status"`
State string `json:"state"`
Nodes []*Node `json:"nodes"`
}
// handlePostQuery handles /query requests.
@ -351,31 +399,19 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
}
func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) {
var ms map[string]uint64
if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse {
ms = h.Holder.MaxInverseSlices()
} else {
ms = h.Holder.MaxSlices()
// handleGetSlicesMax handles GET /schema requests.
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.Holder.MaxSlices(),
Inverse: h.Holder.MaxInverseSlices(),
}); err != nil {
h.logger().Printf("write slices-max response error: %s", err)
}
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
pb := &internal.MaxSlicesResponse{
MaxSlices: ms,
}
if buf, err := proto.Marshal(pb); err != nil {
h.logger().Printf("protobuf marshal error: %s", err)
} else if _, err := w.Write(buf); err != nil {
h.logger().Printf("stream write error: %s", err)
}
return
}
json.NewEncoder(w).Encode(sliceMaxResponse{
MaxSlices: ms,
})
}
type sliceMaxResponse struct {
MaxSlices map[string]uint64 `json:"maxSlices"`
type getSlicesMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
Inverse map[string]uint64 `json:"inverse"`
}
// handleGetIndexes handles GET /index request.
@ -932,7 +968,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
return
}
schema, err := frame.GetFields()
fields, err := frame.GetFields()
if err == ErrFrameFieldsNotAllowed {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -942,7 +978,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
}
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: schema.Fields}); err != nil {
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
@ -1233,9 +1269,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1303,9 +1339,9 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), req.Index, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, req.Index, req.Slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1369,9 +1405,9 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), index, slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.URI, index, slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
if !h.Cluster.OwnsFragment(h.Node.ID, index, slice) {
msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, index, slice)
http.Error(w, msg, http.StatusPreconditionFailed)
return
}
@ -1420,7 +1456,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentBackup handles GET /fragment/data requests.
// handleGetFragmentData handles GET /fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
@ -1443,7 +1479,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
}
}
// handlePostFragmentRestore handles POST /fragment/data requests.
// handlePostFragmentData handles POST /fragment/data requests.
func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
@ -1481,7 +1517,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentData handles GET /fragment/block/data requests.
// handleGetFragmentBlockData handles GET /fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
// Read request object.
var req internal.BlockDataRequest
@ -1597,7 +1633,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
// Loop over each slice and import it if this node owns it.
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
// Ignore this slice if we don't own it.
if !h.Cluster.OwnsFragment(h.URI.HostPort(), indexName, slice) {
if !h.Cluster.OwnsFragment(h.Node.ID, indexName, slice) {
continue
}
@ -1981,6 +2017,131 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) {
}
}
// handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request.
func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req setCoordinatorRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
oldNode := h.Cluster.nodeByID(h.Cluster.Coordinator)
newNode := h.Cluster.nodeByID(req.ID)
if newNode == nil {
http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest)
return
}
if err := func() error {
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == h.Node.ID {
return h.Cluster.SetCoordinator(newNode)
}
// Send the set-coordinator message to new node.
err := h.Broadcaster.SendTo(
newNode,
&internal.SetCoordinatorMessage{
New: EncodeNode(newNode),
})
if err != nil {
return fmt.Errorf("problem sending SetCoordinator message: %s", err)
}
return nil
}(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(setCoordinatorResponse{
Old: oldNode,
New: newNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type setCoordinatorRequest struct {
ID string `json:"id"`
}
type setCoordinatorResponse struct {
Old *Node `json:"old"`
New *Node `json:"new"`
}
// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request.
func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req removeNodeRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
removeNode := h.Cluster.nodeByID(req.ID)
if removeNode == nil {
http.Error(w, fmt.Sprintf("Node is not a member of the cluster: %s", req.ID), http.StatusBadRequest)
return
}
// Start the resize process (similar to NodeJoin)
err = h.Cluster.NodeLeave(removeNode)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(removeNodeResponse{
Remove: removeNode,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type removeNodeRequest struct {
ID string `json:"id"`
}
type removeNodeResponse struct {
Remove *Node `json:"remove"`
}
// handlePostClusterResizeAbort handles POST /cluster/resize/abort request.
func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) {
var msg string
if err := func() error {
if !h.Cluster.IsCoordinator() {
return fmt.Errorf("abort requests must be made on the coordinator node")
}
err := h.Cluster.CompleteCurrentJob(ResizeJobStateAborted)
if err != nil {
return err
}
return nil
}(); err != nil {
msg = err.Error()
}
// Encode response.
if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{
Info: msg,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type clusterResizeAbortResponse struct {
Info string `json:"info"`
}
// InputJSONDataParser validates input json file and executes SetBit.
func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) {
inputDef, err := index.InputDefinition(name)
@ -2051,6 +2212,12 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
}
func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) {
err := h.Broadcaster.SendSync(&internal.RecalculateCaches{})
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
return
}
h.Holder.RecalculateCaches()
w.WriteHeader(http.StatusNoContent)
}
@ -2078,7 +2245,6 @@ func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error)
func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
fmt.Println("**unsupported media type**")
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
}
@ -2110,7 +2276,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
}
func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(h.Holder.LocalID))
_, err := w.Write([]byte(h.Cluster.Node.ID))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}

62
handler_test.go
View file

@ -31,6 +31,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/test"
)
@ -106,6 +107,7 @@ func TestHandler_Schema(t *testing.T) {
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"rowLabel":"rowID","inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"rowLabel":"rowID","cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -146,11 +148,30 @@ func TestHandler_Status(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" {
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.SetRestricted()
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
}
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
@ -171,7 +192,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -212,7 +233,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -1055,9 +1076,6 @@ func TestHandler_Frame_GetFields(t *testing.T) {
t.Fatal(err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f/fields")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
@ -1195,8 +1213,8 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `[{"scheme":"http","host":"host2"},{"scheme":"http","host":"host0"}]`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
// invalid argument should return BadRequest
@ -1828,3 +1846,31 @@ func TestHandler_RecalculateCaches(t *testing.T) {
}
}
func TestHandler_WebUI(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
if !strings.Contains(w.Body.String(), "<title>Pilosa WebUI</title>") {
t.Fatalf("WebUI is not being served correctly.")
}
// If curl is the client, the response should be different
w = httptest.NewRecorder()
req := test.MustNewHTTPRequest("GET", "/", nil)
req.Header.Add("User-Agent", "curl/7.54.0")
h.ServeHTTP(w, req)
if !strings.Contains(w.Body.String(), "try the WebUI") {
t.Fatalf("WebUI is not being served correctly.")
}
}

232
holder.go
View file

@ -31,6 +31,7 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/internal"
uuid "github.com/satori/go.uuid"
)
@ -48,8 +49,15 @@ type Holder struct {
// Indexes by name.
indexes map[string]*Index
hasData bool
// opened channel is closed once Open() completes.
opened chan struct{}
Broadcaster Broadcaster
NewAttrStore func(string) AttrStore
// Close management
wg sync.WaitGroup
closing chan struct{}
@ -64,8 +72,6 @@ type Holder struct {
CacheFlushInterval time.Duration
LogOutput io.Writer
LocalID string
}
// NewHolder returns a new instance of Holder.
@ -74,19 +80,54 @@ func NewHolder() *Holder {
indexes: make(map[string]*Index),
closing: make(chan struct{}, 0),
opened: make(chan struct{}),
Broadcaster: NopBroadcaster,
Stats: NopStatsClient,
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
LogOutput: os.Stderr,
}
}
// Peek reads the root data directory for the holder
// without actually loading any data into memory.
// HasData is returned, and h.hasData is set.
func (h *Holder) Peek() bool {
h.logger().Printf("peek at holder path: %s", h.Path)
h.hasData = false
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
return false
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return false
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
h.hasData = true
break
}
return h.hasData
}
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.setFileLimit()
h.logger().Printf("open holder path: %s", h.Path)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return err
}
@ -124,14 +165,19 @@ func (h *Holder) Open() error {
}
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
}
h.mu.Lock()
h.indexes[index.Name()] = index
h.mu.Unlock()
}
h.logger().Printf("open holder: complete")
// Periodically flush cache.
h.wg.Add(1)
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
h.Stats.Open()
close(h.opened)
return nil
}
@ -151,6 +197,15 @@ func (h *Holder) Close() error {
return nil
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
func (h *Holder) HasData() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return h.hasData || len(h.indexes) > 0
}
// MaxSlices returns MaxSlice map for all indexes.
func (h *Holder) MaxSlices() map[string]uint64 {
a := make(map[string]uint64)
@ -169,13 +224,13 @@ func (h *Holder) MaxInverseSlices() map[string]uint64 {
return a
}
// Schema returns schema data for all indexes and frames.
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name()}
fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
}
@ -189,6 +244,50 @@ func (h *Holder) Schema() []*IndexInfo {
return a
}
// ApplySchema applies an internal Schema to Holder.
func (h *Holder) ApplySchema(schema *internal.Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
opt := IndexOptions{}
idx, err := h.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return err
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
if err != nil {
return err
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
if err != nil {
return err
}
}
}
// TODO: Create inputDefinitions that don't exist.
}
return nil
}
// EncodeMaxSlices creates and internal representation of max slices.
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}
// EncodeSchema creates an internal representation of schema.
func (h *Holder) EncodeSchema() *internal.Schema {
return &internal.Schema{
Indexes: EncodeIndexes(h.Indexes()),
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -278,6 +377,8 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
index.LogOutput = h.LogOutput
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.Broadcaster
index.NewAttrStore = h.NewAttrStore
index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data"))
return index, nil
}
@ -432,22 +533,29 @@ func (h *Holder) setFileLimit() {
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
func (h *Holder) loadLocalID() error {
func (h *Holder) loadNodeID() (string, error) {
idPath := path.Join(h.Path, "ID")
localID := ""
localIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
localID = strings.TrimSpace(string(localIDBytes))
} else {
u := uuid.NewV4()
localID = u.String()
err = ioutil.WriteFile(idPath, []byte(localID), 0600)
if err != nil {
return err
}
nodeID := ""
h.logger().Printf("load NodeID: %s", idPath)
if err := os.MkdirAll(h.Path, 0777); err != nil {
return "", err
}
h.LocalID = localID
return nil
nodeIDBytes, err := ioutil.ReadFile(idPath)
if err == nil {
nodeID = strings.TrimSpace(string(nodeIDBytes))
} else if os.IsNotExist(err) {
nodeID = uuid.NewV4().String()
err = ioutil.WriteFile(idPath, []byte(nodeID), 0600)
if err != nil {
return "", err
}
} else if err != nil {
return "", err
}
return nodeID, nil
}
// HolderSyncer is an active anti-entropy tool that compares the local holder
@ -455,7 +563,7 @@ func (h *Holder) loadLocalID() error {
type HolderSyncer struct {
Holder *Holder
URI *URI
Node *Node
Cluster *Cluster
RemoteClient *http.Client
@ -511,7 +619,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.URI.HostPort(), di.Name, slice) {
if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) {
continue
}
@ -553,11 +661,8 @@ func (s *HolderSyncer) syncIndex(index string) error {
s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
@ -567,7 +672,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.Host})
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.ID})
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
@ -586,7 +691,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve index reference.
// Retrieve frame reference.
f := s.Holder.Frame(index, name)
if f == nil {
return nil
@ -602,11 +707,8 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.URI.HostPort()) {
client, err := NewInternalHTTPClient(node.Host, s.RemoteClient)
if err != nil {
return err
}
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
@ -618,7 +720,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.Host})
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@ -658,7 +760,7 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Host: s.URI.HostPort(),
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
@ -669,3 +771,65 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
return nil
}
// HolderCleaner removes fragments and data files that are no longer used.
type HolderCleaner struct {
Node *Node
Holder *Holder
Cluster *Cluster
// Signals that the sync should stop.
Closing <-chan struct{}
}
// IsClosing returns true if the cleaner has been marked to close.
func (c *HolderCleaner) IsClosing() bool {
select {
case <-c.Closing:
return true
default:
return false
}
}
// CleanHolder compares the holder with the cluster state and removes
// any unnecessary fragments and files.
func (c *HolderCleaner) CleanHolder() error {
for _, index := range c.Holder.Indexes() {
// Verify cleaner has not closed.
if c.IsClosing() {
return nil
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
continue
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
return err
}
}
}
}
}
return nil
}
func uint64InSlice(i uint64, s []uint64) bool {
for _, o := range s {
if i == o {
return true
}
}
return false
}

211
holder_test.go
View file

@ -284,6 +284,60 @@ func TestHolder_Open(t *testing.T) {
})
}
func TestHolder_HasData(t *testing.T) {
t.Run("IndexDirectory", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if h.HasData() {
t.Fatal("expected HasData to return false")
}
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
if !h.HasData() {
t.Fatal("expected HasData to return true")
}
})
t.Run("Peek", func(t *testing.T) {
h := test.NewHolder()
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
}
// Create an index directory to indicate data exists.
if err := os.Mkdir(h.IndexPath("test"), 0777); err != nil {
t.Fatal(err)
}
if hasData := h.Peek(); hasData != true {
t.Fatal("expected Peek to return true")
} else if !h.HasData() {
t.Fatal("expected HasData to return true")
}
})
t.Run("Peek at missing directory", func(t *testing.T) {
h := test.NewHolder()
// Ensure that hasData is false when trying to peek into
// a directory that doesn't exist.
h.Path = "bad-path"
if hasData := h.Peek(); hasData != false {
t.Fatal("expected Peek to return false")
} else if h.HasData() {
t.Fatal("expected HasData to return false")
}
})
}
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
@ -334,16 +388,21 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(client)
e.Holder = hldr1.Holder
e.Scheme = cluster.Nodes[1].Scheme
e.Host = cluster.Nodes[1].Host
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].Host = "localhost:0"
cluster.Nodes[1].Host = test.MustParseURLHost(s.URL)
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
t.Fatal(err)
}
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0, hldr1} {
@ -395,13 +454,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr0.Index("y").SetRemoteMaxSlice(3)
// Set up syncer.
uri, err := cluster.Nodes[0].URI()
if err != nil {
t.Fatal(err)
}
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
URI: uri,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: pilosa.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
@ -440,3 +495,141 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
}
}
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
// Create frames on nodes.
for _, hldr := range []*test.Holder{hldr0} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
t.Fatal(err)
}
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(2)
// Keep replication the same and ensure we get the expected results.
cluster.ReplicaN = 2
// Set up cleaner for replication 2.
cleaner2 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner2.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
// Change replication factor to ensure we have fragments to remove.
cluster.ReplicaN = 1
// Set up cleaner for replication 1.
cleaner1 := pilosa.HolderCleaner{
Node: cluster.Nodes[0],
Holder: hldr0.Holder,
Cluster: cluster,
}
if err := cleaner1.CleanHolder(); err != nil {
t.Fatal(err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
if f != nil {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}
}

54
index.go
View file

@ -55,8 +55,10 @@ type Index struct {
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
NewAttrStore func(string) AttrStore
// Column attribute storage and cache.
columnAttrStore *AttrStore
columnAttrStore AttrStore
// InputDefinitions by name.
inputDefinitions map[string]*InputDefinition
@ -83,7 +85,8 @@ func NewIndex(path, name string) (*Index, error) {
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
columnLabel: DefaultColumnLabel,
@ -100,7 +103,7 @@ func (i *Index) Name() string { return i.name }
func (i *Index) Path() string { return i.path }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore }
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (i *Index) SetColumnLabel(v string) error {
@ -256,9 +259,7 @@ func (i *Index) Close() error {
defer i.mu.Unlock()
// Close the attribute store.
if i.columnAttrStore != nil {
i.columnAttrStore.Close()
}
i.columnAttrStore.Close()
// Close all frames.
for _, f := range i.frames {
@ -392,6 +393,20 @@ func (i *Index) Frames() []*Frame {
return a
}
// InputDefinitions returns a list of all inputDefinitions in the index.
func (i *Index) InputDefinitions() []*InputDefinition {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*InputDefinition, 0, len(i.inputDefinitions))
for _, d := range i.inputDefinitions {
a = append(a, d)
}
//sort.Sort(inputDefintionSlice(a)) // TODO
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
@ -440,8 +455,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
if opt.RangeEnabled {
if opt.InverseEnabled {
return nil, ErrInverseRangeNotAllowed
} else if opt.CacheType != "" && opt.CacheType != CacheTypeNone {
return nil, ErrRangeCacheNotAllowed
}
} else {
if len(opt.Fields) > 0 {
@ -494,18 +507,12 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
f.inverseEnabled = opt.InverseEnabled
f.rangeEnabled = opt.RangeEnabled
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
f.rangeEnabled = opt.RangeEnabled
// Set schema & save.
f.schema = &FrameSchema{
Fields: opt.Fields,
}
if err := f.saveSchema(); err != nil {
// Set fields.
f.fields = opt.Fields
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
@ -524,6 +531,7 @@ func (i *Index) newFrame(path, name string) (*Frame, error) {
f.LogOutput = i.LogOutput
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
return f, nil
}
@ -623,12 +631,10 @@ func EncodeIndexes(a []*Index) []*internal.Index {
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
io := d.options()
return &internal.Index{
Name: d.name,
Meta: io.Encode(),
MaxSlice: d.MaxSlice(),
Frames: encodeFrames(d.Frames()),
Name: d.name,
Frames: encodeFrames(d.Frames()),
InputDefinitions: encodeInputDefinitions(d.InputDefinitions()),
}
}
@ -723,7 +729,6 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) {
if err != nil {
return nil, err
}
inputDef.broadcaster = i.broadcaster
return inputDef, nil
}
@ -776,7 +781,6 @@ func (i *Index) openInputDefinitions() error {
return nil
}
}
}
return nil
}

24
index_test.go
View file

@ -106,25 +106,21 @@ func TestIndex_CreateFrame(t *testing.T) {
},
}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema: %#v", f.Schema())
t.Fatalf("unexpected fields: %#v", f.Fields())
}
// Reopen the index & verify the fields are loaded.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Schema(), &pilosa.FrameSchema{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected schema after reopen: %#v", f.Schema())
t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
}
})
@ -140,14 +136,14 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) {
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
CacheType: pilosa.CacheTypeRanked,
}); err != pilosa.ErrRangeCacheNotAllowed {
}); err != nil {
t.Fatal(err)
}
})

60
input_definition.go
View file

@ -36,12 +36,11 @@ var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleR
// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
name string
path string
index string
broadcaster Broadcaster
frames []InputFrame
fields []InputDefinitionField
name string
path string
index string
frames []InputFrame
fields []InputDefinitionField
}
// NewInputDefinition returns a new instance of InputDefinition.
@ -86,17 +85,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
// Copy metadata fields.
i.name = pb.Name
for _, fr := range pb.Frames {
frameMeta := fr.Meta
inputFrame := InputFrame{
Name: fr.Name,
Options: FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
RowLabel: DefaultRowLabel,
InverseEnabled: frameMeta.InverseEnabled,
CacheSize: frameMeta.CacheSize,
CacheType: frameMeta.CacheType,
TimeQuantum: TimeQuantum(frameMeta.TimeQuantum),
},
Name: fr.Name,
Options: *decodeFrameOptions(fr.Meta),
}
i.frames = append(i.frames, inputFrame)
}
@ -338,6 +329,43 @@ func (i *InputDefinitionInfo) Encode() *internal.InputDefinition {
return &def
}
// encodeInputDefinitions converts a into its internal representation.
func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition {
other := make([]*internal.InputDefinition, len(a))
for i := range a {
other[i] = encodeInputDefinition(a[i])
}
return other
}
// encodeInputDefinition converts i into its internal representation.
func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition {
//fo := f.options()
return &internal.InputDefinition{
Name: i.name,
Frames: encodeInputFrames(i.frames),
Fields: encodeInputDefinitionFields(i.fields),
}
}
// encodeInputFrames converts a into its internal representation.
func encodeInputFrames(a []InputFrame) []*internal.Frame {
other := make([]*internal.Frame, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// encodeInputDefinitionFields converts a into its internal representation.
func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField {
other := make([]*internal.InputDefinitionField, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// AddFrame manually add frame to input definition.
func (i *InputDefinition) AddFrame(frame InputFrame) error {
i.frames = append(i.frames, frame)

3717
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

94
internal/private.proto
View file

@ -38,8 +38,9 @@ message Cache {
repeated uint64 IDs = 1;
}
message MaxSlicesResponse {
map<string, uint64> MaxSlices = 1;
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
@ -83,14 +84,16 @@ message DeleteFieldMessage {
message Frame {
string Name = 1;
FrameMeta Meta = 2;
repeated string Views = 3;
}
message Schema {
repeated Index Indexes = 1;
}
message Index {
string Name = 1;
IndexMeta Meta = 2;
uint64 MaxSlice = 3;
repeated Frame Frames = 4;
repeated uint64 Slices = 5;
repeated InputDefinition InputDefinitions = 6;
}
@ -124,19 +127,38 @@ message DeleteInputDefinitionMessage {
string Name = 2;
}
message NodeStatus {
string Host = 1;
message URI {
string Scheme = 1;
string Host = 2;
uint32 Port = 3;
}
message Node {
string ID = 1;
URI URI = 2;
bool IsCoordinator = 3;
}
message NodeStateMessage {
string NodeID = 1;
string State = 2;
repeated Index Indexes = 3;
string Scheme = 4;
}
message NodeEventMessage {
uint32 Event = 1;
Node Node = 2;
}
message NodeStatus {
Node Node = 1;
MaxSlices MaxSlices = 2;
Schema Schema = 3;
}
message ClusterStatus {
repeated NodeStatus Nodes = 1;
}
message FrameSchema {
repeated Field Fields = 1;
string ClusterID = 1;
string State = 2;
repeated Node Nodes = 3;
}
message Field {
@ -146,8 +168,52 @@ message Field {
int64 Max = 4;
}
message CreateViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}
message ResizeInstruction {
int64 JobID = 1;
Node Node = 2;
Node Coordinator = 3;
repeated ResizeSource Sources = 4;
Schema Schema = 5;
ClusterStatus ClusterStatus = 6;
}
message ResizeSource {
Node Node = 1;
string Index = 2;
string Frame = 3;
string View = 4;
uint64 Slice = 5;
}
message ResizeInstructionComplete {
int64 JobID = 1;
Node Node = 2;
string Error = 3;
}
message SetCoordinatorMessage {
Node New = 1;
}
message UpdateCoordinatorMessage {
Node New = 1;
}
message Topology {
string ClusterID = 1;
repeated string NodeIDs = 2;
}
message RecalculateCaches {}

47
pilosa.go
View file

@ -73,8 +73,7 @@ var (
ErrQueryRequired = errors.New("query required")
ErrTooManyWrites = errors.New("too many write commands")
ErrConfigClusterTypeInvalid = errors.New("invalid cluster type")
ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts")
ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed")
)
// Regular expression to validate index and frame names.
@ -162,6 +161,50 @@ func StringInSlice(a string, list []string) bool {
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {

32
security_manager.go Normal file
View file

@ -0,0 +1,32 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
// SecurityManager provides the ability to limit access to restricted endpoints
// during cluster configuration.
type SecurityManager interface {
SetRestricted()
SetNormal()
}
// NopSecurityManager provides a no-op implementation of the SecurityManager interface.
type NopSecurityManager struct {
}
// SetRestricted no-op.
func (sdm *NopSecurityManager) SetRestricted() {}
// SetNormal no-op.
func (sdm *NopSecurityManager) SetNormal() {}

575
server.go
View file

@ -23,7 +23,6 @@ import (
"log"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"runtime"
@ -32,17 +31,15 @@ import (
"sync"
"time"
"github.com/CAFxX/gcnotifier"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/diagnostics"
"github.com/pilosa/pilosa/internal"
"golang.org/x/sync/errgroup"
)
// Default server settings.
const (
DefaultAntiEntropyInterval = 10 * time.Minute
DefaultPollingInterval = 60 * time.Second
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
)
@ -68,16 +65,19 @@ type Server struct {
RemoteClient *http.Client
// Cluster configuration.
// Host is replaced with actual host after opening if port is ":0".
Network string
URI *URI
NodeID string
URI URI
Cluster *Cluster
diagnostics *diagnostics.Diagnostics
ClusterID string
diagnostics *DiagnosticsCollector
SystemInfo SystemInfo
GCNotifier GCNotifier
NewAttrStore func(string) AttrStore
// Background monitoring intervals.
AntiEntropyInterval time.Duration
PollingInterval time.Duration
MetricInterval time.Duration
DiagnosticInterval time.Duration
@ -102,12 +102,16 @@ func NewServer() *Server {
Handler: NewHandler(),
Broadcaster: NopBroadcaster,
BroadcastReceiver: NopBroadcastReceiver,
diagnostics: diagnostics.New(DefaultDiagnosticServer),
diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
SystemInfo: NewNopSystemInfo(),
Network: "tcp",
GCNotifier: NopGCNotifier,
NewAttrStore: NewNopAttrStore,
AntiEntropyInterval: DefaultAntiEntropyInterval,
PollingInterval: DefaultPollingInterval,
MetricInterval: 0,
DiagnosticInterval: 0,
@ -116,11 +120,117 @@ func NewServer() *Server {
s.logger = log.New(s.LogOutput, "", log.LstdFlags)
s.Handler.Holder = s.Holder
s.diagnostics.server = s
return s
}
// Open opens and initializes the server.
func (s *Server) Open() error {
s.Logger().Printf("open server")
// s.ln can be configured prior to Open() via s.OpenListener().
if s.ln == nil {
if err := s.OpenListener(); err != nil {
return err
}
}
// Get or create NodeID.
s.NodeID = s.LoadNodeID()
// Set Cluster Node.
node := &Node{
ID: s.NodeID,
URI: s.URI,
IsCoordinator: s.Cluster.Coordinator == s.NodeID,
}
s.Cluster.Node = node
// Append the NodeID tag to stats.
s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID))
// Peek at the holder to determine if there is data on disk.
// Don't actually load the data until after the Cluster
// management starts.
s.Holder.LogOutput = s.LogOutput
s.Holder.Peek()
// Create default HTTP client
s.createDefaultClient(s.RemoteClient)
// Create executor for executing queries.
e := NewExecutor(s.RemoteClient)
e.Holder = s.Holder
e.Node = node
e.Cluster = s.Cluster
e.MaxWritesPerRequest = s.MaxWritesPerRequest
// Cluster settings.
s.Cluster.Broadcaster = s.Broadcaster
s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
// Initialize HTTP handler.
s.Handler.Broadcaster = s.Broadcaster
s.Handler.BroadcastHandler = s
s.Handler.StatusHandler = s
s.Handler.Node = node
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Handler.LogOutput = s.LogOutput
s.Cluster.prefect = s.Handler
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
err := http.Serve(s.ln, s.Handler)
if err != nil {
s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
}
// Open Cluster management.
if err := s.Cluster.Open(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
}
// Open holder.
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.Cluster.SetNodeState(NodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
// Listen for joining nodes.
// This needs to start after the Holder has opened so that nodes can join
// the cluster without waiting for data to load on the coordinator. Before
// this starts, the joins are queued up in the Cluster.joiningLeavingNodes
// buffered channel.
s.Cluster.ListenForJoins()
// Start background monitoring.
s.wg.Add(3)
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
return nil
}
// OpenListener opens a listener for the Server.
func (s *Server) OpenListener() error {
s.Logger().Printf("open server listener: %s", s.URI)
if s.ln != nil {
return fmt.Errorf("a listener already exists for server: %s", s.URI)
}
var ln net.Listener
var err error
@ -148,87 +258,6 @@ func (s *Server) Open() error {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
// Create local node if no cluster is specified.
if len(s.Cluster.Nodes) == 0 {
s.Cluster.Nodes = []*Node{
{Scheme: s.URI.Scheme(), Host: s.URI.HostPort()},
}
}
for i, n := range s.Cluster.Nodes {
if s.Cluster.NodeByHost(n.Host) != nil {
s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%d", i))
}
}
// Open holder.
s.Holder.LogOutput = s.LogOutput
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
}
// Open NodeSet communication
if err := s.Cluster.NodeSet.Open(); err != nil {
return fmt.Errorf("opening NodeSet: %v", err)
}
// Create default HTTP client
s.createDefaultClient(s.RemoteClient)
// Create executor for executing queries.
e := NewExecutor(s.RemoteClient)
e.Holder = s.Holder
e.Scheme = s.URI.Scheme()
e.Host = s.URI.HostPort()
e.Cluster = s.Cluster
e.MaxWritesPerRequest = s.MaxWritesPerRequest
s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest
// Initialize HTTP handler.
s.Handler.Broadcaster = s.Broadcaster
s.Handler.BroadcastHandler = s
s.Handler.StatusHandler = s
s.Handler.URI = s.URI
s.Handler.Cluster = s.Cluster
s.Handler.Executor = e
s.Handler.LogOutput = s.LogOutput
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
server := &http.Server{Handler: s.Handler}
go func() {
<-s.closing
server.Close()
}()
err := server.Serve(ln)
if err != nil && err.Error() != "http: Server closed" {
s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// load local ID
if err := s.Holder.loadLocalID(); err != nil {
s.Logger().Println(err)
}
if err := s.loadClusterID(); err != nil {
s.Logger().Println(err)
}
// Start background monitoring.
s.wg.Add(4)
go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
go func() { defer s.wg.Done(); s.monitorMaxSlices() }()
go func() { defer s.wg.Done(); s.monitorRuntime() }()
go func() { defer s.wg.Done(); s.monitorDiagnostics() }()
return nil
}
@ -241,6 +270,9 @@ func (s *Server) Close() error {
if s.ln != nil {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.Close()
}
if s.Holder != nil {
s.Holder.Close()
}
@ -248,6 +280,20 @@ func (s *Server) Close() error {
return nil
}
// LoadNodeID gets NodeID from disk, or creates a new value.
// If server.NodeID is already set, a new ID is not created.
func (s *Server) LoadNodeID() string {
if s.NodeID != "" {
return s.NodeID
}
nodeID, err := s.Holder.loadNodeID()
if err != nil {
s.Logger().Printf("loading NodeID: %v", err)
return s.NodeID
}
return nodeID
}
// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
if s.ln == nil {
@ -298,7 +344,7 @@ func (s *Server) monitorAntiEntropy() {
// Initialize syncer with local holder and remote client.
var syncer HolderSyncer
syncer.Holder = s.Holder
syncer.URI = s.URI
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.RemoteClient
@ -317,44 +363,6 @@ func (s *Server) monitorAntiEntropy() {
}
}
// monitorMaxSlices periodically pulls the highest slice from each node in the cluster.
func (s *Server) monitorMaxSlices() {
// Ignore if only one node in the cluster.
if len(s.Cluster.Nodes) <= 1 {
return
}
ticker := time.NewTicker(s.PollingInterval)
defer ticker.Stop()
for {
select {
case <-s.closing:
return
case <-ticker.C:
}
oldmaxslices := s.Holder.MaxSlices()
for _, node := range s.Cluster.Nodes {
if s.URI.HostPort() != node.Host {
maxSlices, _ := s.checkMaxSlices(node.Scheme, node.Host)
for index, newmax := range maxSlices {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex := s.Holder.Index(index); localIndex != nil {
if newmax > oldmaxslices[index] {
oldmaxslices[index] = newmax
localIndex.SetRemoteMaxSlice(newmax)
}
} else {
s.Logger().Printf("Local Index not found: %s", index)
}
}
}
}
}
}
// ReceiveMessage represents an implementation of BroadcastHandler.
func (s *Server) ReceiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
@ -386,16 +394,8 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := FrameOptions{
RowLabel: obj.Meta.RowLabel,
InverseEnabled: obj.Meta.InverseEnabled,
RangeEnabled: obj.Meta.RangeEnabled,
CacheType: obj.Meta.CacheType,
CacheSize: obj.Meta.CacheSize,
TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum),
Fields: decodeFields(obj.Meta.Fields),
}
_, err := idx.CreateFrame(obj.Frame, opt)
opt := decodeFrameOptions(obj.Meta)
_, err := idx.CreateFrame(obj.Frame, *opt)
if err != nil {
return err
}
@ -427,6 +427,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.CreateViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
}
_, _, err := f.createViewIfNotExistsBase(obj.View)
if err != nil {
return err
}
case *internal.DeleteViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {
@ -436,7 +445,36 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.ClusterStatus:
err := s.Cluster.MergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
err := s.Cluster.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
err := s.Cluster.MarkResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.Cluster.SetCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.Cluster.UpdateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
case *internal.RecalculateCaches:
s.Holder.RecalculateCaches()
case *internal.NodeEventMessage:
s.Cluster.ReceiveEvent(DecodeNodeEvent(obj))
}
return nil
}
@ -444,17 +482,13 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
func (s *Server) SendSync(pb proto.Message) error {
var eg errgroup.Group
for _, node := range s.Cluster.Nodes {
uri, err := node.URI()
if err != nil {
return err
}
s.Logger().Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if *s.URI == *uri {
if s.URI == node.URI {
continue
}
ctx := context.WithValue(context.Background(), "uri", uri)
ctx := context.WithValue(context.Background(), "uri", &node.URI)
eg.Go(func() error {
return s.defaultClient.SendMessage(ctx, pb)
})
@ -468,150 +502,145 @@ func (s *Server) SendAsync(pb proto.Message) error {
return s.Gossiper.SendAsync(pb)
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
s.Logger().Printf("SendTo: %s", to.URI)
ctx := context.WithValue(context.Background(), "uri", &to.URI)
return s.defaultClient.SendMessage(ctx, pb)
}
// Server implements StatusHandler.
// LocalStatus is used to periodically sync information
// between nodes. Under normal conditions, nodes should
// remain in sync through Broadcast messages. For cases
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice/MaxInverseSlice by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Cluster == nil {
return nil, errors.New("Server.Cluster is nil")
}
if s.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Scheme: s.URI.Scheme(),
Host: s.URI.HostPort(),
State: NodeStateUp,
Indexes: EncodeIndexes(s.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.URI.HostPort())
Node: EncodeNode(s.Cluster.Node),
MaxSlices: s.Holder.EncodeMaxSlices(),
Schema: s.Holder.EncodeSchema(),
}
return &ns, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
// ClusterStatus returns the ClusterState and NodeSet for the cluster.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Update local Node.state.
ns, err := s.LocalStatus()
if err != nil {
return nil, err
}
node := s.Cluster.NodeByHost(s.URI.HostPort())
node.SetStatus(ns.(*internal.NodeStatus))
// Update NodeState for all nodes.
for host, nodeState := range s.Cluster.NodeStates() {
// In a default configuration (or single-node) where a StaticNodeSet is used
// then all nodes are marked as DOWN. At the very least, we should consider
// the local node as UP.
// TODO: we should be able to remove this check if/when cluster.Nodes and
// cluster.NodeSet are unified.
if host == s.URI.HostPort() {
nodeState = NodeStateUp
}
node := s.Cluster.NodeByHost(host)
node.SetState(nodeState)
}
return s.Cluster.Status(), nil
}
// HandleRemoteStatus receives incoming NodeState from remote nodes.
// HandleRemoteStatus receives incoming NodeStatus from remote nodes.
func (s *Server) HandleRemoteStatus(pb proto.Message) error {
return s.mergeRemoteStatus(pb.(*internal.NodeStatus))
// Ignore NodeStatus messages until the cluster is in a Normal state.
if s.Cluster.State() != ClusterStateNormal {
return nil
}
go func() {
// Make sure the holder has opened.
<-s.Holder.opened
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
if err != nil {
s.Logger().Printf("merge remote status: %s", err)
}
}()
return nil
}
func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// Update Node.state.
node := s.Cluster.NodeByHost(ns.Host)
node.SetStatus(ns)
// Ignore status updates from self.
if s.NodeID == DecodeNode(ns.Node).ID {
return nil
}
// Create indexes that don't exist.
for _, index := range ns.Indexes {
opt := IndexOptions{
ColumnLabel: index.Meta.ColumnLabel,
TimeQuantum: TimeQuantum(index.Meta.TimeQuantum),
// Sync schema.
if err := s.Holder.ApplySchema(ns.Schema); err != nil {
return err
}
// Sync maxSlices (standard).
oldmaxslices := s.Holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
continue
}
idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return err
if newMax > oldmaxslices[index] {
oldmaxslices[index] = newMax
localIndex.SetRemoteMaxSlice(newMax)
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := FrameOptions{
RowLabel: f.Meta.RowLabel,
TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),
CacheSize: f.Meta.CacheSize,
}
_, err := idx.CreateFrameIfNotExists(f.Name, opt)
if err != nil {
return err
}
}
// Sync maxSlices (inverse).
oldMaxInverseSlices := s.Holder.MaxInverseSlices()
for index, newMaxInverse := range ns.MaxSlices.Inverse {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.Logger().Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
oldMaxInverseSlices[index] = newMaxInverse
localIndex.SetRemoteMaxInverseSlice(newMaxInverse)
}
}
return nil
}
func (s *Server) checkMaxSlices(scheme string, hostPort string) (map[string]uint64, error) {
// Create HTTP request.
req, err := http.NewRequest("GET", (&url.URL{
Scheme: scheme,
Host: hostPort,
Path: "/slices/max",
}).String(), nil)
if err != nil {
return nil, err
}
// Require protobuf encoding.
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("User-Agent", "pilosa/"+Version)
nodeURI, err := NewURIFromAddress(hostPort)
if err != nil {
return nil, err
}
nodeURI.SetScheme(scheme)
ctx := context.WithValue(context.Background(), "uri", nodeURI)
return s.defaultClient.MaxSliceByIndex(ctx)
}
// monitorDiagnostics periodically polls the Pilosa Indexes for cluster info.
func (s *Server) monitorDiagnostics() {
if s.DiagnosticInterval <= 0 {
// Do not send more than once a minute
if s.DiagnosticInterval < time.Minute {
s.Logger().Printf("diagnostics disabled")
return
} else {
s.Logger().Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval)
}
s.diagnostics.SetLogger(s.LogOutput)
s.diagnostics.SetVersion(Version)
s.diagnostics.SetInterval(s.DiagnosticInterval)
s.diagnostics.Open()
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeSetHosts(), ","))
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("LocalID", s.Holder.LocalID)
s.diagnostics.Set("ClusterID", s.ClusterID)
s.diagnostics.Set("NodeID", s.NodeID)
s.diagnostics.Set("ClusterID", s.Cluster.ID)
s.diagnostics.EnrichWithOSInfo()
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder)
openFiles, err := CountOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
s.diagnostics.Set("GoRoutines", runtime.NumGoroutine())
s.diagnostics.EnrichWithMemoryInfo()
s.diagnostics.EnrichWithSchemaProperties()
s.diagnostics.CheckVersion()
s.diagnostics.Flush()
err = s.diagnostics.Flush()
if err != nil {
s.Logger().Printf("Diagnostics error: %s", err)
}
}
ticker := time.NewTicker(s.DiagnosticInterval)
@ -639,8 +668,7 @@ func (s *Server) monitorRuntime() {
ticker := time.NewTicker(s.MetricInterval)
defer ticker.Stop()
gcn := gcnotifier.New()
defer gcn.Close()
defer s.GCNotifier.Close()
s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
@ -649,7 +677,7 @@ func (s *Server) monitorRuntime() {
select {
case <-s.closing:
return
case <-gcn.AfterGC():
case <-s.GCNotifier.AfterGC():
// GC just ran.
s.Holder.Stats.Count("garbage_collection", 1, 1.0)
case <-ticker.C:
@ -678,25 +706,6 @@ func (s *Server) createDefaultClient(remoteClient *http.Client) {
s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient)
}
func (s *Server) loadClusterID() error {
// If this is the first node in the cluster, set the ClusterID to its ID
node0URI, err := s.Cluster.Nodes[0].URI()
if err == nil {
if s.URI.Equals(node0URI) {
s.ClusterID = s.Holder.LocalID
return nil
}
} else {
return err
}
if clusterID, err := s.defaultClient.NodeID(node0URI); err == nil {
s.ClusterID = clusterID
return nil
} else {
return err
}
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
switch runtime.GOOS {
@ -718,47 +727,11 @@ func CountOpenFiles() (int, error) {
}
}
// StatusHandler specifies two methods which an object must implement to share
// state in the cluster. These are used by the GossipNodeSet to implement the
// StatusHandler specifies the methods which an object must implement to share
// state in the cluster. These are used by the GossipMemberSet to implement the
// LocalState and MergeRemoteState methods of memberlist.Delegate
type StatusHandler interface {
LocalStatus() (proto.Message, error)
ClusterStatus() (proto.Message, error)
HandleRemoteStatus(proto.Message) error
}
type diagnosticsFrameProperties struct {
BSIFieldCount int
TimeQuantumEnabled bool
}
func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) {
// NOTE: this function is not in the diagnostics package, since circular imports are not allowed.
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields.Fields)
}
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}

581
server/cluster_test.go Normal file
View file

@ -0,0 +1,581 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server_test
import (
"context"
"fmt"
"net/http"
"reflect"
"strings"
"testing"
"time"
"golang.org/x/sync/errgroup"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/test"
)
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
m0 := test.MustRunMain()
defer m0.Close()
m1 := test.MustRunMain()
defer m1.Close()
// Update cluster config
m0.Server.Cluster.Nodes = []*pilosa.Node{
{ID: m0.Server.NodeID, URI: m0.Server.URI},
{ID: m1.Server.NodeID, URI: m1.Server.URI},
}
m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
// Configure node0
// get the host portion of addr to use for binding
m0.Config.Gossip.Port = "0"
m0.Config.Gossip.Seeds = []string{}
m0.Server.Cluster.Coordinator = m0.Server.NodeID
m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}}
m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.LogOutput)
gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server)
if err != nil {
t.Fatal(err)
}
m0.Server.Cluster.MemberSet = gossipMemberSet0
m0.Server.Broadcaster = m0.Server
m0.Server.Gossiper = gossipMemberSet0
m0.Server.Handler.Broadcaster = m0.Server.Broadcaster
m0.Server.Holder.Broadcaster = m0.Server.Broadcaster
m0.Server.BroadcastReceiver = gossipMemberSet0
if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m0.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
// Configure node1
// get the host portion of addr to use for binding
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()}
m1.Server.Cluster.Coordinator = m0.Server.NodeID
m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.LogOutput)
gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server)
if err != nil {
t.Fatal(err)
}
m1.Server.Cluster.MemberSet = gossipMemberSet1
m1.Server.Broadcaster = m1.Server
m1.Server.Gossiper = gossipMemberSet1
m1.Server.Handler.Broadcaster = m1.Server.Broadcaster
m1.Server.Holder.Broadcaster = m1.Server.Broadcaster
m1.Server.BroadcastReceiver = gossipMemberSet1
if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil {
t.Fatal(err)
}
// Open Cluster management.
if err := m1.Server.Cluster.Open(); err != nil {
t.Fatal(err)
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Expected indexes and Frames
expected := map[string][]string{
"i": []string{"f"},
}
// Create a client for each node.
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Make sure node0 knows about the index and frame created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
}
// Make sure node1 knows about the index and frame created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
}
}
if !reflect.DeepEqual(received1, expected) {
t.Fatalf("unexpected schema on node1: %s", received1)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
// Make sure node0 knows about the latest MaxSlice.
maxSlices0, err := client0.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices0["i"] != 2 {
t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"])
}
// Make sure node1 knows about the latest MaxSlice.
maxSlices1, err := client1.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
}
// Write input definition to the first node.
if _, err := m0.CreateDefinition("i", "test", `{
"frames": [{"name": "event-time",
"options": {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "columnID",
"primaryKey": true
}]}
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
frame0 := m0.Server.Holder.Frame("i", "event-time")
if frame0 == nil {
t.Fatal("frame not found")
}
frame1 := m1.Server.Holder.Frame("i", "event-time")
if frame1 == nil {
t.Fatal("frame not found")
}
}
// Ensure that an empty node comes up in a NORMAL state.
func TestClusterResize_EmptyNode(t *testing.T) {
m0 := test.MustRunMain()
defer m0.Close()
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State())
}
}
// Ensure that a cluster of empty nodes comes up in a NORMAL state.
func TestClusterResize_EmptyNodes(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
gossipHost := "localhost"
gossipPort := 0
seed, err := m0.RunWithTransport(gossipHost, gossipPort, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
seed, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed})
if err != nil {
t.Fatal(err)
}
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
}
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNode(t *testing.T) {
t.Run("NoData", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("WithIndex", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("ContinuousSlices", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=1300000)
`); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
t.Run("SkippedSlice", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Create a client for each node.
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
_, err = m1.RunWithTransport("localhost", 0, []string{seed})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
}
})
}
// Ensure that redundant gossip seeds are used
func TestCluster_GossipMembership(t *testing.T) {
t.Run("Node0Down", func(t *testing.T) {
// Configure node0
m0 := test.NewMainWithCluster(true)
defer m0.Close()
seed, err := m0.RunWithTransport("localhost", 0, []string{})
if err != nil {
t.Fatal(err)
}
// Configure node1
m1 := test.NewMainWithCluster(false)
defer m1.Close()
var eg errgroup.Group
eg.Go(func() error {
// Pass invalid seed as first in list
_, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed})
if err != nil {
return err
}
return nil
})
// Configure node2
m2 := test.NewMainWithCluster(false)
defer m2.Close()
eg.Go(func() error {
// Pass invalid seed as last in list
_, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"})
if err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
t.Fatal(err)
}
// Give the cluster time to settle.
time.Sleep(1 * time.Second)
if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
} else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal {
t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State())
}
numNodes := len(m0.Server.Cluster.Status().Nodes)
if numNodes != 3 {
t.Fatalf("Expected 3 nodes, got %d", numNodes)
}
})
}
func TestClusterResize_RemoveNode(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3)
m0 := cluster[0]
m1 := cluster[1]
t.Run("ErrorRemoveInvalidNode", func(t *testing.T) {
resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`)
expBody := "Node is not a member of the cluster: invalid-node-id"
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveCoordinator", func(t *testing.T) {
resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator."
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveOnNonCoordinator", func(t *testing.T) {
resp := test.MustDo("GET", m0.URL()+fmt.Sprintf("/id"), "")
coordinatorNodeID := resp.Body
resp = test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID)
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if strings.TrimSpace(resp.Body) != expBody {
t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) {
client0 := m0.Client()
// Create indexes and frames on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// This is an attempt to ensure there is data on both nodes, but is not guaranteed.
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(rowID=1, frame=\"f\", columnID=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {
t.Fatal(err)
}
resp := test.MustDo("GET", m1.URL()+fmt.Sprintf("/id"), "")
nodeID := resp.Body
resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID))
expBody := "not enough data to perform resize"
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode)
} else if !strings.Contains(resp.Body, expBody) {
t.Fatalf("expected to contain '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body))
}
})
}

165
server/server.go
View file

@ -32,10 +32,12 @@ import (
"crypto/tls"
"io/ioutil"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/statsd"
)
@ -59,10 +61,13 @@ type Command struct {
CPUProfile string
CPUTime time.Duration
// Gossip transport
GossipTransport *gossip.Transport
// Standard input/output
*pilosa.CmdIO
// running will be closed once Command.Run is finished.
// Started will be closed once Command.Run is finished.
Started chan struct{}
// Done will be closed when Command.Close() is called
Done chan struct{}
@ -99,12 +104,18 @@ func (m *Command) Run(args ...string) (err error) {
return err
}
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
return err
}
// Initialize server.
if err = m.Server.Open(); err != nil {
return fmt.Errorf("server.Open: %v", err)
}
m.Server.Logger().Printf("Listening as %s\n", m.Server.URI.Normalize())
m.Server.Logger().Printf("Listening as %s\n", m.Server.URI)
return nil
}
@ -116,24 +127,16 @@ func (m *Command) SetupServer() error {
}
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
return err
}
m.Server.URI = uri
m.Server.URI = *uri
cluster := pilosa.NewCluster()
cluster.ReplicaN = m.Config.Cluster.ReplicaN
cluster.Holder = m.Server.Holder
for _, address := range m.Config.Cluster.Hosts {
uri, err := pilosa.NewURIFromAddress(address)
if err != nil {
return err
}
cluster.Nodes = append(cluster.Nodes, &pilosa.Node{
Scheme: uri.Scheme(),
Host: uri.HostPort(),
})
}
m.Server.Cluster = cluster
// Setup logging output.
@ -142,6 +145,12 @@ func (m *Command) SetupServer() error {
return err
}
// Configure data directory (for Cluster .topology)
m.Server.Cluster.Path = m.Config.DataDir
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = boltdb.NewAttrStore
// Configure holder.
m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Holder.Path = m.Config.DataDir
@ -149,6 +158,8 @@ func (m *Command) SetupServer() error {
if m.Config.Metric.Diagnostics {
m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval)
}
m.Server.SystemInfo = gopsutil.NewSystemInfo()
m.Server.GCNotifier = gcnotify.NewActiveGCNotifier()
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
if err != nil {
return err
@ -177,68 +188,15 @@ func (m *Command) SetupServer() error {
InsecureSkipVerify: m.Config.TLS.SkipVerify,
}
// TODO Review this location
TLSConfig = m.Server.TLS
}
c := pilosa.GetHTTPClient(TLSConfig)
m.Server.RemoteClient = c
m.Server.Handler.RemoteClient = c
m.Server.Cluster.RemoteClient = c
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
// Config.GossipPort is deprecated, so Config.Gossip.Port has priority
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
} else if m.Config.GossipPort != "" {
gossipPortStr = m.Config.GossipPort
}
switch m.Config.Cluster.Type {
case pilosa.ClusterGossip:
gossipPort, err := strconv.Atoi(gossipPortStr)
if err != nil {
return err
}
gossipSeed := pilosa.DefaultHost + ":" + pilosa.DefaultGossipPort
// Config.GossipSeed is deprecated, so Config.Gossip.Seed has priority
if m.Config.Gossip.Seed != "" {
gossipSeed = m.Config.Gossip.Seed
} else if m.Config.GossipSeed != "" {
gossipSeed = m.Config.GossipSeed
}
var gossipKey []byte
if m.Config.Gossip.Key != "" {
gossipKey, err = ioutil.ReadFile(m.Config.Gossip.Key)
if err != nil {
return err
}
}
// get the host portion of addr to use for binding
gossipHost := uri.Host()
gossipNodeSet, err := gossip.NewGossipNodeSet(uri.HostPort(), gossipHost, gossipPort, gossipSeed, m.Server, gossipKey)
if err != nil {
return err
}
m.Server.Cluster.NodeSet = gossipNodeSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipNodeSet
m.Server.Gossiper = gossipNodeSet
case pilosa.ClusterStatic, pilosa.ClusterNone:
m.Server.Broadcaster = pilosa.NopBroadcaster
m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
m.Server.Gossiper = pilosa.NopGossiper
err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
if err != nil {
return err
}
default:
return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type)
}
// Statik file system.
m.Server.Handler.FileSystem = &statik.FileSystem{}
// Set configuration options.
m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval)
@ -246,6 +204,71 @@ func (m *Command) SetupServer() error {
return nil
}
// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {
m.Server.NodeID = m.Server.LoadNodeID()
if m.Config.Cluster.Disabled {
m.Server.Cluster.Static = true
m.Server.Cluster.Coordinator = m.Server.NodeID
for _, address := range m.Config.Cluster.Hosts {
uri, err := pilosa.NewURIFromAddress(address)
if err != nil {
return err
}
m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{
URI: *uri,
})
}
m.Server.Broadcaster = pilosa.NopBroadcaster
m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes)
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
m.Server.Gossiper = pilosa.NopGossiper
return nil
}
// Set internal port (string).
gossipPortStr := pilosa.DefaultGossipPort
if m.Config.Gossip.Port != "" {
gossipPortStr = m.Config.Gossip.Port
}
gossipPort, err := strconv.Atoi(gossipPortStr)
if err != nil {
return err
}
// get the host portion of addr to use for binding
gossipHost := m.Server.URI.Host()
var transport *gossip.Transport
if m.GossipTransport != nil {
transport = m.GossipTransport
} else {
transport, err = gossip.NewTransport(gossipHost, gossipPort)
if err != nil {
return err
}
}
// Set Coordinator.
if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 {
m.Server.Cluster.Coordinator = m.Server.NodeID
}
m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.LogOutput)
gossipMemberSet, err := gossip.NewGossipMemberSetWithTransport(m.Server.NodeID, m.Config, transport, m.Server)
if err != nil {
return err
}
m.Server.Cluster.MemberSet = gossipMemberSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipMemberSet
m.Server.Gossiper = gossipMemberSet
return nil
}
// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path.
func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) {
// This is split out so it can be used in NewServeCmd as well as SetupServer

323
server/server_test.go
View file

@ -15,26 +15,20 @@
package server_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
"reflect"
"runtime"
"sort"
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -45,7 +39,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
if err := quick.Check(func(cmds []SetCommand) bool {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create client.
@ -121,7 +115,7 @@ func TestMain_Set_Quick(t *testing.T) {
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -198,7 +192,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -242,7 +236,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Ensure program can set column attributes with columnLabel option.
func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
m := MustRunMain()
m := test.MustRunMain()
defer m.Close()
// Create frames.
@ -276,11 +270,12 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := NewMainArrayWithCluster(2)
m0 := mains1[0]
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
m11 := mains1[1]
// Create frames.
client := m0.Client()
client := m10.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
@ -289,7 +284,7 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Write data on first cluster.
if _, err := m0.Query("i", "", `
if _, err := m10.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=100)
SetBit(rowID=1, frame="f", columnID=1000)
SetBit(rowID=1, frame="f", columnID=100000)
@ -302,34 +297,45 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Query row on first cluster.
if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m10.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Start second cluster.
mains2 := NewMainArrayWithCluster(2)
m2 := mains2[0]
defer m2.Close()
mains2 := test.MustRunMainWithCluster(t, 2)
m20 := mains2[0]
defer m20.Close()
m21 := mains2[1]
defer m21.Close()
// Import from first cluster.
client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create new index:", err)
}
if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create new frame:", err)
}
if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil {
if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
// Query row on second cluster.
if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
if res, err := m20.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
@ -378,226 +384,45 @@ func TestCountOpenFiles(t *testing.T) {
}
}
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
mains := NewMainArrayWithCluster(2)
m0 := mains[0]
defer m0.Close()
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)
m1 := mains[1]
defer m1.Close()
// Expected indexes and Frames
expected := map[string][]string{
"i": []string{"f"},
}
// Create a client for each node.
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
// Create the schema.
client0 := cluster[0].Client()
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
t.Fatal("create index:", err)
}
if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create frame:", err)
}
// Make sure node0 knows about the index and frame created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
// Set some bits
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(rowID=%d, frame="f", columnID=%d)`, rowID, columnID))
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
t.Fatal("setting bits:", err)
}
// Make sure node1 knows about the index and frame created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
}
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
// Calculate caches on the first node
cluster[0].RecalculateCaches()
target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}`
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(frame="f")`)
if err != nil {
t.Fatal(err)
}
res = strings.TrimSpace(res)
if sortedString(target) != sortedString(res) {
t.Fatalf("%v != %v", target, res)
}
}
if !reflect.DeepEqual(received1, expected) {
t.Fatalf("unexpected schema on node1: %s", received1)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=1)
SetBit(rowID=1, frame="f", columnID=2400000)
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
// Make sure node0 knows about the latest MaxSlice.
maxSlices0, err := client0.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices0["i"] != 2 {
t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"])
}
// Make sure node1 knows about the latest MaxSlice.
maxSlices1, err := client1.MaxSliceByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
}
// Write input definition to the first node.
if _, err := m0.CreateDefinition("i", "test", `{
"frames": [{"name": "event-time",
"options": {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "columnID",
"primaryKey": true
}]}
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
frame0 := m0.Server.Holder.Frame("i", "event-time")
if frame0 == nil {
t.Fatal("frame not found")
}
frame1 := m1.Server.Holder.Frame("i", "event-time")
if frame1 == nil {
t.Fatal("frame not found")
}
}
// Main represents a test wrapper for main.Main.
type Main struct {
*server.Command
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *test.Network
m.Config.DataDir = path
m.Config.Bind = "localhost:0"
m.Config.Cluster.Type = "static"
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
}
return m
}
func NewMainArrayWithCluster(size int) []*Main {
cluster, err := test.NewServerCluster(size)
if err != nil {
panic(err)
}
mainArray := make([]*Main, size)
for i := 0; i < size; i++ {
mainArray[i] = &Main{Command: cluster.Servers[i]}
}
return mainArray
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
if err := m.Run(); err != nil {
panic(err)
}
return m
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
// Create new main with the same config.
config := m.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
m.Server.Network = *test.Network
m.Config = config
// Run new program.
if err := m.Run(); err != nil {
return err
}
return nil
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
panic(err)
}
return client
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// CreateDefinition.
func (m *Main) CreateDefinition(index, def, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// SetCommand represents a command to set a bit.
@ -656,32 +481,6 @@ func ParseConfig(s string) (pilosa.Config, error) {
return c, err
}
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.
func MustDo(method, urlStr string, body string) *httpResponse {
req, err := http.NewRequest(method, urlStr, strings.NewReader(body))
if err != nil {
panic(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
return &httpResponse{Response: resp, Body: string(buf)}
}
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*http.Response
Body string
}
// MustMarshalJSON marshals v into a string. Panic on error.
func MustMarshalJSON(v interface{}) string {
buf, err := json.Marshal(v)
@ -691,6 +490,12 @@ func MustMarshalJSON(v interface{}) string {
return string(buf)
}
func sortedString(s string) string {
arr := strings.Split(s, "")
sort.Strings(arr)
return strings.Join(arr, "")
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

2
statik/.gitignore vendored
View file

@ -1 +1 @@
statik.go
/statik.go

3
statik/doc.go
View file

@ -1,3 +0,0 @@
// Package statik contains static assets for the Web UI. `go generate` will
// produce statik.go, which is ignored by git.
package statik

37
statik/filesystem.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//go:generate statik -src=../webui -dest=..
//
// Package statik contains static assets for the Web UI. `go generate` or
// `make generate-statik` will produce statik.go, which is ignored by git.
package statik
import (
"net/http"
"github.com/pilosa/pilosa"
"github.com/rakyll/statik/fs"
)
// Ensure nopFileSystem implements interface.
var _ pilosa.FileSystem = &FileSystem{}
// FileSystem represents a static FileSystem.
type FileSystem struct{}
// New is a statik implementation of FileSystem New method.
func (s *FileSystem) New() (http.FileSystem, error) {
return fs.New()
}

11
test/attr.go
View file

@ -22,15 +22,16 @@ import (
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
*pilosa.AttrStore
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore() *AttrStore {
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
@ -38,7 +39,7 @@ func NewAttrStore() *AttrStore {
f.Close()
os.Remove(f.Name())
return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
@ -74,8 +75,8 @@ func BenchmarkAttrStore_Duplicate(b *testing.B) {
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() *AttrStore {
s := NewAttrStore()
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}

398
test/cluster.go
View file

@ -15,24 +15,42 @@
package test
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
path, err := ioutil.TempDir("", "pilosa-cluster-")
if err != nil {
panic(err)
}
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Scheme: "http",
Host: fmt.Sprintf("host%d", i),
ID: fmt.Sprintf("node%d", i),
URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
c.Node = c.Nodes[0]
c.Coordinator = c.Nodes[0].ID
return c
}
@ -53,3 +71,379 @@ type ConstHasher struct {
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// NewURI is a test URI creator that intentionally swallows errors.
func NewURI(scheme, host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewURIFromHostPort(host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// TestCluster represents a cluster of test nodes, each of which
// has a pilosa.Cluster.
type TestCluster struct {
Clusters []*pilosa.Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*pilosa.Node
}
func (t *TestCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / pilosa.SliceWidth
nodes := c0.FragmentNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *TestCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &pilosa.NodeEvent{
Event: pilosa.NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != pilosa.ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &pilosa.Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := pilosa.NewHolder()
h.Path = path
// cluster
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
c.Holder = h
c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the TestCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewTestCluster returns a new instance of test.Cluster.
func NewTestCluster(n int) *TestCluster {
tc := &TestCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *TestCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *TestCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(pilosa.NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *TestCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// TestCluster implements Broadcaster interface.
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *TestCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == pilosa.ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *TestCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := pilosa.DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

5
test/executor.go
View file

@ -34,14 +34,13 @@ func init() {
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
// The executor always matches the uri of the first cluster node.
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
executor := pilosa.NewExecutor(remoteClient)
e := &Executor{Executor: executor}
e.Holder = holder
e.Cluster = cluster
e.Scheme = cluster.Nodes[0].Scheme
e.Host = cluster.Nodes[0].Host
e.Node = cluster.Nodes[0]
return e
}

6
test/fragment.go
View file

@ -27,7 +27,7 @@ const SliceWidth = pilosa.SliceWidth
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
RowAttrStore *AttrStore
RowAttrStore pilosa.AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
@ -43,7 +43,7 @@ func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fra
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
return f
}
@ -78,7 +78,7 @@ func (f *Fragment) Reopen() error {
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
return err
}

48
test/handler.go
View file

@ -17,7 +17,6 @@ package test
import (
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
@ -47,6 +46,8 @@ func NewHandler() *Handler {
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
h.SetNormal()
return h
}
@ -80,43 +81,36 @@ func NewServer() *Server {
if err != nil {
panic(err)
}
s.Handler.URI = uri
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Cluster.Nodes[0].URI = *uri
s.Handler.Node = s.Handler.Cluster.Nodes[0]
return s
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
// LocalStatus exists so that test.Server implements StatusHandler.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Handler.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Host: s.Handler.Handler.URI.HostPort(),
State: pilosa.NodeStateUp,
Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.URI.HostPort())
}
return &ns, nil
return nil, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
// ClusterStatus exists so that test.Server implements StatusHandler.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Assuming we are only testing this with one Node
// So just return its status
return s.LocalStatus()
id := "test-node"
uri := pilosa.DefaultURI()
node := &pilosa.Node{
ID: id,
URI: *uri,
}
return &internal.ClusterStatus{
ClusterID: "",
State: pilosa.ClusterStateNormal,
Nodes: pilosa.EncodeNodes([]*pilosa.Node{node}),
}, nil
}
// HandleRemoteStatus just need to implement a nop to complete the Interface
@ -125,12 +119,12 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil }
// Host returns the hostname of the running server.
func (s *Server) Host() string { return MustParseURLHost(s.URL) }
func (s *Server) HostURI() *pilosa.URI {
func (s *Server) HostURI() pilosa.URI {
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
panic(err)
}
return uri
return *uri
}
// MustParseURLHost parses rawurl and returns the hostname. Panic on error.

3
test/holder.go
View file

@ -20,6 +20,7 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// Holder is a test wrapper for pilosa.Holder.
@ -38,6 +39,7 @@ func NewHolder() *Holder {
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h
}
@ -64,6 +66,7 @@ func (h *Holder) Reopen() error {
h.Holder = pilosa.NewHolder()
h.Holder.Path = path
h.Holder.LogOutput = logOutput
h.Holder.NewAttrStore = boltdb.NewAttrStore
if err := h.Holder.Open(); err != nil {
return err
}

352
test/pilosa.go
View file

@ -16,140 +16,268 @@ package test
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net"
"strconv"
"net/http"
"os"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
func MustNewRunningServer(t *testing.T) *server.Command {
s, err := newServer()
if err != nil {
t.Fatalf("getting new server: %v", err)
}
err = s.Run()
if err != nil {
t.Fatalf("running new pilosa server: %v", err)
}
return s
}
func newServer() (*server.Command, error) {
s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
port, err := findPort()
if err != nil {
return nil, errors.Wrap(err, "getting port")
}
s.Config.Bind = "localhost:" + strconv.Itoa(port)
gport, err := findPort()
if err != nil {
return nil, errors.Wrap(err, "getting gossip port")
}
s.Config.GossipPort = strconv.Itoa(gport)
s.Config.GossipSeed = "localhost:" + s.Config.GossipPort
s.Config.Cluster.Type = "gossip"
td, err := ioutil.TempDir("", "")
if err != nil {
return nil, errors.Wrap(err, "temp dir")
}
s.Config.DataDir = td
return s, nil
}
func findPort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", ":0")
if err != nil {
return 0, errors.Wrap(err, "resolving new port addr")
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, errors.Wrap(err, "listening to get new port")
}
port := l.Addr().(*net.TCPAddr).Port
err = l.Close()
if err != nil {
return port, errors.Wrap(err, "closing listener")
}
return port, nil
}
func MustFindPort(t *testing.T) int {
port, err := findPort()
if err != nil {
t.Fatalf("allocating new port: %v", err)
}
return port
}
type Cluster struct {
Servers []*server.Command
}
func MustNewServerCluster(t *testing.T, size int) *Cluster {
cluster, err := NewServerCluster(size)
if err != nil {
t.Fatalf("new cluster: %v", err)
}
return cluster
}
// **** below exists to have interface compatibility with cluster-resize,
// **** we can remove when cluster resize gets merged *****************//
type M struct {
////////////////////////////////////////////////////////////////////////////////////
// Main represents a test wrapper for main.Main.
type Main struct {
*server.Command
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
func MustRunMainWithCluster(t *testing.T, size int) []*M {
cluster := MustNewServerCluster(t, size)
mains := make([]*M, 0)
for _, s := range cluster.Servers {
mains = append(mains, &M{Command: s})
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
return mains
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *Network
m.Server.NewAttrStore = NewAttrStore
m.Server.Holder.NewAttrStore = NewAttrStore
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
}
return m
}
// ***********************************************************************************//
// NewMainWithCluster returns a new instance of Main with clustering enabled.
func NewMainWithCluster(isCoordinator bool) *Main {
m := NewMain()
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
}
func NewServerCluster(size int) (cluster *Cluster, err error) {
cluster = &Cluster{
Servers: make([]*server.Command, size),
// MustRunMainWithCluster ruturns a running array of *Main where
// all nodes are joined via memberlist (i.e. clustering enabled).
func MustRunMainWithCluster(t *testing.T, size int) []*Main {
ma, err := runMainWithCluster(size)
if err != nil {
t.Fatalf("new main array with cluster: %v", err)
}
hosts := make([]string, size)
return ma
}
// runMainWithCluster runs an array of *Main where all nodes are
// joined via memberlist (i.e. clustering enabled).
func runMainWithCluster(size int) ([]*Main, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
mains := make([]*Main, size)
gossipHost := "localhost"
gossipPort := 0
var err error
var gossipSeeds = make([]string, size)
for i := 0; i < size; i++ {
s, err := newServer()
m := NewMainWithCluster(i == 0)
gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i])
if err != nil {
return nil, errors.Wrap(err, "new server")
return nil, errors.Wrap(err, "RunWithTransport")
}
cluster.Servers[i] = s
hosts[i] = s.Config.Bind
s.Config.GossipSeed = cluster.Servers[0].Config.GossipSeed
mains[i] = m
}
for _, s := range cluster.Servers {
s.Config.Cluster.Hosts = hosts
}
for i, s := range cluster.Servers {
err := s.Run()
if err != nil {
for j := 0; j <= i; j++ {
cluster.Servers[j].Close()
}
return nil, errors.Wrapf(err, "starting server %d of %d. Config: %#v", i+1, size, s.Config)
}
}
return cluster, nil
return mains, nil
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Run(); err != nil {
panic(err)
}
return m
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
// Create new main with the same config.
config := m.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr)
m.Server.Network = *Network
m.Server.NewAttrStore = boltdb.NewAttrStore
m.Server.Holder.NewAttrStore = m.Server.NewAttrStore
m.Config = config
// Run new program.
if err := m.Run(); err != nil {
return err
}
return nil
}
// RunWithTransport runs Main and returns the dynamically allocated gossip port.
func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (seed string, err error) {
defer close(m.Started)
/*
TEST:
- SetupServer (just static settings from config)
- OpenListener (sets Server.Name to use in gossip)
- NewTransport (gossip)
- SetupNetworking (does the gossip or static stuff) - uses Server.Name
- Open server
PRODUCTION:
- SetupServer (just static settings from config)
- SetupNetworking (does the gossip or static stuff) - calls NewTransport
- Open server - calls OpenListener
*/
// SetupServer
err = m.SetupServer()
if err != nil {
return seed, err
}
// Open server listener.
err = m.Server.OpenListener()
if err != nil {
return seed, err
}
// Open gossip transport to use in SetupServer.
transport, err := gossip.NewTransport(host, bindPort)
if err != nil {
return seed, err
}
m.GossipTransport = transport
if len(joinSeeds) != 0 {
m.Config.Gossip.Seeds = joinSeeds
} else {
m.Config.Gossip.Seeds = []string{transport.URI.String()}
}
seed = transport.URI.String()
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
return seed, err
}
if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil {
return seed, err
}
m.Server.Cluster.Static = false
// Initialize server.
err = m.Server.Open()
if err != nil {
return seed, err
}
return seed, nil
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
panic(err)
}
return client
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
// CreateDefinition.
func (m *Main) CreateDefinition(index, def, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
func (m *Main) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
if resp.StatusCode != 204 {
return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return nil
}
////////////////////////////////////////////////////////////////////////////////////
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.
func MustDo(method, urlStr string, body string) *httpResponse {
req, err := http.NewRequest(method, urlStr, strings.NewReader(body))
if err != nil {
panic(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
return &httpResponse{Response: resp, Body: string(buf)}
}
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*http.Response
Body string
}

35
test/pilosa_test.go
View file

@ -15,28 +15,35 @@
package test_test
import (
"encoding/json"
"net/http"
"testing"
"encoding/json"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestNewCluster(t *testing.T) {
cluster := test.MustNewServerCluster(t, 3)
response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status")
numNodes := 3
cluster := test.MustRunMainWithCluster(t, numNodes)
coordinator := cluster[0].Server.Cluster.Coordinator
for i := 1; i < numNodes; i++ {
if coordi := cluster[i].Server.Cluster.Coordinator; coordi != coordinator {
t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator)
}
}
response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status")
if err != nil {
t.Fatalf("getting schema: %v", err)
}
dec := json.NewDecoder(response.Body)
body := struct {
Status struct {
Nodes []struct {
Host string
Schema string
State string
}
State string
Nodes []struct {
Scheme string
Host string
Port int
}
}{}
@ -50,13 +57,11 @@ func TestNewCluster(t *testing.T) {
t.Fatalf("encoding: %v", err)
}
if len(body.Status.Nodes) != 3 {
if len(body.Nodes) != 3 {
t.Fatalf("wrong number of nodes in status: %s", bytes)
}
for i, node := range body.Status.Nodes {
if node.State != "UP" {
t.Fatalf("node %d should be up but is %s", i, node.State)
}
if body.State != pilosa.ClusterStateNormal {
t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State)
}
}

108
uri.go
View file

@ -15,11 +15,14 @@
package pilosa
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/pilosa/pilosa/internal"
)
var schemeRegexp = regexp.MustCompile("^[+a-z]+$")
@ -40,9 +43,9 @@ var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0-
// localhost
// :10101
type URI struct {
scheme string
host string
port uint16
scheme string `json:"scheme"`
host string `json:"host"`
port uint16 `json:"port"`
}
// DefaultURI creates and returns the default URI.
@ -54,6 +57,16 @@ func DefaultURI() *URI {
}
}
type URIs []URI
func (u URIs) HostPortStrings() []string {
s := make([]string, len(u))
for i, a := range u {
s[i] = a.HostPort()
}
return s
}
// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
uri := DefaultURI()
@ -67,11 +80,7 @@ func NewURIFromHostPort(host string, port uint16) (*URI, error) {
// NewURIFromAddress parses the passed address and returns a URI.
func NewURIFromAddress(address string) (*URI, error) {
uri, err := parseAddress(address)
if err != nil {
return nil, err
}
return uri, err
return parseAddress(address)
}
// Scheme returns the scheme of this URI.
@ -144,9 +153,7 @@ func (u URI) Equals(other *URI) bool {
if other == nil {
return false
}
return u.scheme == other.scheme &&
u.host == other.host &&
u.port == other.port
return u == *other
}
// Path returns URI with path
@ -198,3 +205,82 @@ func parseAddress(address string) (uri *URI, err error) {
}
return uri, nil
}
// Encode converts o into its internal representation.
func (u URI) Encode() *internal.URI {
return encodeURI(u)
}
func encodeURI(u URI) *internal.URI {
return &internal.URI{
Scheme: u.scheme,
Host: u.host,
Port: uint32(u.port),
}
}
func DecodeURI(i *internal.URI) URI {
return decodeURI(i)
}
func decodeURI(i *internal.URI) URI {
if i == nil {
return URI{}
}
return URI{
scheme: i.Scheme,
host: i.Host,
port: uint16(i.Port),
}
}
func encodeURIs(a []URI) []*internal.URI {
if len(a) == 0 {
return nil
}
other := make([]*internal.URI, len(a))
for i := range a {
other[i] = encodeURI(a[i])
}
return other
}
func decodeURIs(a []*internal.URI) []URI {
if len(a) == 0 {
return nil
}
other := make([]URI, len(a))
for i := range a {
other[i] = decodeURI(a[i])
}
return other
}
// MarshalJSON marshals URI into a JSON-encoded byte slice.
func (u *URI) MarshalJSON() ([]byte, error) {
var output struct {
Scheme string `json:"scheme,omitempty"`
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
output.Scheme = u.scheme
output.Host = u.host
output.Port = u.port
return json.Marshal(output)
}
func (u *URI) UnmarshalJSON(b []byte) error {
var input struct {
Scheme string `json:"scheme,omitempty"`
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
if err := json.Unmarshal(b, &input); err != nil {
return err
}
u.scheme = input.Scheme
u.host = input.Host
u.port = input.Port
return nil
}

42
view.go
View file

@ -18,6 +18,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
@ -62,7 +63,7 @@ type View struct {
broadcaster Broadcaster
stats StatsClient
RowAttrStore *AttrStore
RowAttrStore AttrStore
LogOutput io.Writer
}
@ -98,6 +99,12 @@ func (v *View) Path() string { return v.path }
// Open opens and initializes the view.
func (v *View) Open() error {
// Never keep a cache for field views.
if strings.HasPrefix(v.name, ViewFieldPrefix) {
v.cacheType = CacheTypeNone
}
if err := func() error {
// Ensure the view's path exists.
if err := os.MkdirAll(v.path, 0777); err != nil {
@ -119,6 +126,9 @@ func (v *View) Open() error {
return nil
}
// logger returns a logger instance for the view.
func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) }
// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
@ -270,6 +280,36 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
return frag
}
// DeleteFragment removes the fragment from the view.
func (v *View) DeleteFragment(slice uint64) error {
fragment := v.fragments[slice]
if fragment == nil {
return ErrFragmentNotFound
}
v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return err
}
// Delete fragment file.
if err := os.Remove(fragment.Path()); err != nil {
return err
}
// Delete fragment cache file.
if err := os.Remove(fragment.CachePath()); err != nil {
v.logger().Printf("no cache file to delete for slice %d", slice)
}
delete(v.fragments, slice)
return nil
}
// SetBit sets a bit within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth

45
view_test.go
View file

@ -17,6 +17,7 @@ package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
@ -25,22 +26,21 @@ import (
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
RowAttrStore *test.AttrStore
RowAttrStore pilosa.AttrStore
}
// NewView returns a new instance of View with a temporary path.
func NewView(index, frame, name string) *View {
file, err := ioutil.TempFile("", "pilosa-view-")
path, err := ioutil.TempDir("", "pilosa-view-")
if err != nil {
panic(err)
}
file.Close()
v := &View{
View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize),
View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize),
RowAttrStore: test.MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore
return v
}
@ -68,7 +68,7 @@ func (v *View) Reopen() error {
}
v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View.RowAttrStore = v.RowAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore
if err := v.Open(); err != nil {
return err
}
@ -93,3 +93,36 @@ func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
}
}
}
// Ensure view can open and retrieve a fragment.
func TestView_DeleteFragment(t *testing.T) {
v := MustOpenView("i", "f", "v")
defer v.Close()
slice := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.DeleteFragment(slice)
if err != nil {
t.Fatal(err)
}
if v.Fragment(slice) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same slice, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {
t.Fatal("failed to create new fragment")
}
}