scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 07:01:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into issue-1283

Browse source
This commit is contained in:
tgruben 2018-06-18 14:20:21 -05:00 • committed by GitHub
commit f412f4d0c7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
58 changed files with 4404 additions and 1848 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
Gopkg.lock generated
View file

@ -70,6 +70,18 @@
packages = ["proto"]
revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9"
[[projects]]
name = "github.com/google/go-cmp"
packages = [
"cmp",
"cmp/cmpopts",
"cmp/internal/diff",
"cmp/internal/function",
"cmp/internal/value"
]
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
version = "v0.2.0"
[[projects]]
name = "github.com/gorilla/context"
packages = ["."]
@ -304,6 +316,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4"
inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5"
solver-name = "gps-cdcl"
solver-version = 1

33
api.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
@ -45,7 +44,7 @@ type API struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
Cluster *Cluster
RemoteClient *http.Client
TranslateStore TranslateStore
Logger Logger
}
@ -126,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if api.TranslateStore != nil {
for _, col := range resp.ColumnAttrSets {
v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
return resp, nil
@ -291,7 +302,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return ErrClusterDoesNotOwnSlice
}
@ -327,7 +338,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64)
return nil, errors.Wrap(err, "validating api method")
}
return api.Cluster.SliceNodes(indexName, slice), nil
return api.Cluster.sliceNodes(indexName, slice), nil
}
// MarshalFragment returns an object which can write the specified fragment's data
@ -681,7 +692,7 @@ func (api *API) LongQueryTime() time.Duration {
func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) {
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return nil, nil, ErrClusterDoesNotOwnSlice
}
@ -709,15 +720,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
return nil, nil, errors.Wrap(err, "validating api method")
}
oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.NodeByID(id)
oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.nodeByID(id)
if newNode == nil {
return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node")
}
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == api.LocalID() {
return oldNode, newNode, api.Cluster.SetCoordinator(newNode)
return oldNode, newNode, api.Cluster.setCoordinator(newNode)
}
// Send the set-coordinator message to new node.
@ -739,13 +750,13 @@ func (api *API) RemoveNode(id string) (*Node, error) {
return nil, errors.Wrap(err, "validating api method")
}
removeNode := api.Cluster.nodeByID(id)
removeNode := api.Cluster.unprotectedNodeByID(id)
if removeNode == nil {
return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
}
// Start the resize process (similar to NodeJoin)
err := api.Cluster.NodeLeave(removeNode)
err := api.Cluster.nodeLeave(removeNode)
if err != nil {
return removeNode, errors.Wrap(err, "calling node leave")
}
@ -758,7 +769,7 @@ func (api *API) ResizeAbort() error {
return errors.Wrap(err, "validating api method")
}
err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted)
err := api.Cluster.completeCurrentJob(resizeJobStateAborted)
return errors.Wrap(err, "complete current job")
}

26
attr.go
View file

@ -24,10 +24,10 @@ import (
// Attribute data type enum.
const (
AttrTypeString = 1
AttrTypeInt = 2
AttrTypeBool = 3
AttrTypeFloat = 4
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
// AttrStore represents an interface for handling row/column attributes.
@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
pb := &internal.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = AttrTypeString
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = AttrTypeFloat
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = AttrTypeBool
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
// decodeAttr converts from an Attr internal representation to a key/value pair.
func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
switch attr.Type {
case AttrTypeString:
case attrTypeString:
return attr.Key, attr.StringValue
case AttrTypeInt:
case attrTypeInt:
return attr.Key, attr.IntValue
case AttrTypeBool:
case attrTypeBool:
return attr.Key, attr.BoolValue
case AttrTypeFloat:
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil

79
attr_test.go
View file

@ -15,15 +15,20 @@
package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) {
// Ensure database returns a non-nil empty map if unset.
func TestAttrStore_Attrs_Empty(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
if m, err := s.Attrs(100); err != nil {
@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
// Ensure database can unset attributes if explicitly set to nil.
func TestAttrStore_Attrs_Unset(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
// Ensure attribute block checksums can be returned.
func TestAttrStore_Blocks(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) {
t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
}
}
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

90
broadcast.go
View file

@ -120,21 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error {
// Broadcast message types.
const (
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
messageTypeCreateSlice = iota
messageTypeCreateIndex
messageTypeDeleteIndex
messageTypeCreateField
messageTypeDeleteField
messageTypeCreateView
messageTypeDeleteView
messageTypeClusterStatus
messageTypeResizeInstruction
messageTypeResizeInstructionComplete
messageTypeSetCoordinator
messageTypeUpdateCoordinator
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -142,35 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateSliceMessage:
typ = MessageTypeCreateSlice
typ = messageTypeCreateSlice
case *internal.CreateIndexMessage:
typ = MessageTypeCreateIndex
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = MessageTypeDeleteIndex
typ = messageTypeDeleteIndex
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
typ = messageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
typ = messageTypeDeleteField
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
typ = messageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
typ = messageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
typ = messageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
typ = messageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
typ = messageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
typ = messageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
typ = messageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
typ = messageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
typ = messageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
typ = messageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -187,35 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
var m proto.Message
switch typ {
case MessageTypeCreateSlice:
case messageTypeCreateSlice:
m = &internal.CreateSliceMessage{}
case MessageTypeCreateIndex:
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
case MessageTypeDeleteIndex:
case messageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
case MessageTypeCreateField:
case messageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
case messageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateView:
case messageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
case messageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
case messageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
case messageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
case messageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
case messageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
case messageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
case messageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
case messageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
case messageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)

6
cache.go
View file

@ -27,8 +27,8 @@ import (
)
const (
// ThresholdFactor is used to calculate the threshold for new items entering the cache
ThresholdFactor = 1.1
// thresholdFactor is used to calculate the threshold for new items entering the cache
thresholdFactor = 1.1
)
// Cache represents a cache of counts.
@ -158,7 +158,7 @@ type RankCache struct {
func NewRankCache(maxEntries uint32) *RankCache {
return &RankCache{
maxEntries: maxEntries,
thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
entries: make(map[uint64]uint64),
stats: NopStatsClient,
}

426
cluster.go
View file

@ -21,7 +21,6 @@ import (
"hash/fnv"
"io/ioutil"
"math/rand"
"net/http"
"os"
"path/filepath"
"sort"
@ -49,14 +48,14 @@ const (
NodeStateLoading = "LOADING"
NodeStateReady = "READY"
// ResizeJob states.
ResizeJobStateRunning = "RUNNING"
// resizeJob states.
resizeJobStateRunning = "RUNNING"
// Final states.
ResizeJobStateDone = "DONE"
ResizeJobStateAborted = "ABORTED"
resizeJobStateDone = "DONE"
resizeJobStateAborted = "ABORTED"
ResizeJobActionAdd = "ADD"
ResizeJobActionRemove = "REMOVE"
resizeJobActionAdd = "ADD"
resizeJobActionRemove = "REMOVE"
)
// Node represents a node in the cluster.
@ -255,8 +254,8 @@ type Cluster struct {
joined bool
mu sync.RWMutex
jobs map[int64]*ResizeJob
currentJob *ResizeJob
jobs map[int64]*resizeJob
currentJob *resizeJob
// Close management
wg sync.WaitGroup
@ -264,9 +263,6 @@ type Cluster struct {
Logger Logger
//
RemoteClient *http.Client
InternalClient InternalClient
}
@ -279,7 +275,7 @@ func NewCluster() *Cluster {
EventReceiver: NopEventReceiver,
joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
jobs: make(map[int64]*ResizeJob),
jobs: make(map[int64]*resizeJob),
closing: make(chan struct{}),
joining: make(chan struct{}),
@ -289,27 +285,27 @@ func NewCluster() *Cluster {
}
}
// Coordinator returns the coordinator node.
func (c *Cluster) CoordinatorNode() *Node {
return c.nodeByID(c.Coordinator)
// coordinatorNode returns the coordinator node.
func (c *Cluster) coordinatorNode() *Node {
return c.unprotectedNodeByID(c.Coordinator)
}
// IsCoordinator is true if this node is the coordinator.
func (c *Cluster) IsCoordinator() bool {
// isCoordinator is true if this node is the coordinator.
func (c *Cluster) isCoordinator() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.isCoordinator()
return c.unprotectedIsCoordinator()
}
func (c *Cluster) isCoordinator() bool {
func (c *Cluster) unprotectedIsCoordinator() bool {
return c.Coordinator == c.Node.ID
}
// SetCoordinator tells the current node to become the
// setCoordinator tells the current node to become the
// Coordinator. In response to this, the current node
// will consider itself coordinator and update the other
// nodes with its version of Cluster.Status.
func (c *Cluster) SetCoordinator(n *Node) error {
func (c *Cluster) setCoordinator(n *Node) error {
c.mu.Lock()
// Verify that the new Coordinator value matches
// this node.
@ -319,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error {
}
// Update IsCoordinator on all nodes (locally).
_ = c.updateCoordinator(n)
_ = c.unprotectedUpdateCoordinator(n)
c.mu.Unlock()
// Send the update coordinator message to all nodes.
err := c.Broadcaster.SendSync(
@ -334,17 +330,17 @@ func (c *Cluster) SetCoordinator(n *Node) error {
return c.Broadcaster.SendSync(c.Status())
}
// UpdateCoordinator updates this nodes Coordinator value as well as
// updateCoordinator updates this nodes Coordinator value as well as
// changing the corresponding node's IsCoordinator value
// to true, and sets all other nodes to false. Returns true if the value
// changed.
func (c *Cluster) UpdateCoordinator(n *Node) bool {
func (c *Cluster) updateCoordinator(n *Node) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.updateCoordinator(n)
return c.unprotectedUpdateCoordinator(n)
}
func (c *Cluster) updateCoordinator(n *Node) bool {
func (c *Cluster) unprotectedUpdateCoordinator(n *Node) bool {
var changed bool
if c.Coordinator != n.ID {
c.Coordinator = n.ID
@ -360,9 +356,9 @@ func (c *Cluster) updateCoordinator(n *Node) bool {
return changed
}
// AddNode adds a node to the Cluster and updates and saves the
// addNode adds a node to the Cluster and updates and saves the
// new topology.
func (c *Cluster) AddNode(node *Node) error {
func (c *Cluster) addNode(node *Node) error {
c.Logger.Printf("add node %s to cluster on %s", node, c.Node)
// If the node being added is the coordinator, set it for this node.
@ -387,9 +383,9 @@ func (c *Cluster) AddNode(node *Node) error {
return c.saveTopology()
}
// RemoveNode removes a node from the Cluster and updates and saves the
// removeNode removes a node from the Cluster and updates and saves the
// new topology.
func (c *Cluster) RemoveNode(node *Node) error {
func (c *Cluster) removeNode(node *Node) error {
// remove from cluster
if !c.removeNodeBasicSorted(node) {
return nil
@ -407,8 +403,8 @@ func (c *Cluster) RemoveNode(node *Node) error {
return c.saveTopology()
}
// NodeIDs returns the list of IDs in the cluster.
func (c *Cluster) NodeIDs() []string {
// nodeIDs returns the list of IDs in the cluster.
func (c *Cluster) nodeIDs() []string {
return Nodes(c.Nodes).IDs()
}
@ -472,9 +468,9 @@ func (c *Cluster) setState(state string) {
}
}
func (c *Cluster) SetNodeState(state string) error {
if c.IsCoordinator() {
return c.ReceiveNodeState(c.Node.ID, state)
func (c *Cluster) setNodeState(state string) error {
if c.isCoordinator() {
return c.receiveNodeState(c.Node.ID, state)
}
// Send node state to coordinator.
@ -484,18 +480,18 @@ func (c *Cluster) SetNodeState(state string) error {
}
c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator)
if err := c.sendTo(c.CoordinatorNode(), ns); err != nil {
if err := c.sendTo(c.coordinatorNode(), ns); err != nil {
return fmt.Errorf("sending node state error: err=%s", err)
}
return nil
}
// ReceiveNodeState sets node state in Topology in order for the
// receiveNodeState sets node state in Topology in order for the
// Coordinator to keep track of, during startup, which nodes have
// finished opening their Holder.
func (c *Cluster) ReceiveNodeState(nodeID string, state string) error {
if !c.IsCoordinator() {
func (c *Cluster) receiveNodeState(nodeID string, state string) error {
if !c.isCoordinator() {
return nil
}
@ -515,11 +511,6 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error {
return nil
}
// localNode is not being used.
//func (c *Cluster) localNode() *Node {
// return c.NodeByURI(c.URI)
//}
// Status returns the internal ClusterStatus representation.
func (c *Cluster) Status() *internal.ClusterStatus {
return &internal.ClusterStatus{
@ -529,14 +520,14 @@ func (c *Cluster) Status() *internal.ClusterStatus {
}
}
func (c *Cluster) NodeByID(id string) *Node {
func (c *Cluster) nodeByID(id string) *Node {
c.mu.RLock()
defer c.mu.RUnlock()
return c.nodeByID(id)
return c.unprotectedNodeByID(id)
}
// nodeByID returns a node reference by ID.
func (c *Cluster) nodeByID(id string) *Node {
// unprotectedNodeByID returns a node reference by ID.
func (c *Cluster) unprotectedNodeByID(id string) *Node {
for _, n := range c.Nodes {
if n.ID == id {
return n
@ -558,7 +549,7 @@ func (c *Cluster) nodePositionByID(nodeID string) int {
// addNodeBasicSorted adds a node to the cluster, sorted by id.
// Returns a pointer to the node and true if the node was added.
func (c *Cluster) addNodeBasicSorted(node *Node) bool {
n := c.nodeByID(node.ID)
n := c.unprotectedNodeByID(node.ID)
if n != nil {
return false
}
@ -645,7 +636,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost {
t := make(fragsByHost)
for i := uint64(0); i <= maxSlice; i++ {
nodes := c.SliceNodes(idx, i)
nodes := c.sliceNodes(idx, i)
for _, n := range nodes {
// for each field/view combination:
for field, views := range fieldViews {
@ -673,10 +664,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error)
if lenTo-lenFrom > 1 {
return "", "", errors.New("adding more than one node at a time is not supported")
}
action = ResizeJobActionAdd
action = resizeJobActionAdd
// Determine the node ID that is being added.
for _, n := range other.Nodes {
if c.nodeByID(n.ID) == nil {
if c.unprotectedNodeByID(n.ID) == nil {
nodeID = n.ID
break
}
@ -686,10 +677,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error)
if lenFrom-lenTo > 1 {
return "", "", errors.New("removing more than one node at a time is not supported")
}
action = ResizeJobActionRemove
action = resizeJobActionRemove
// Determine the node ID that is being removed.
for _, n := range c.Nodes {
if other.nodeByID(n.ID) == nil {
if other.unprotectedNodeByID(n.ID) == nil {
nodeID = n.ID
break
}
@ -721,7 +712,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
// If a node is being removed, however, then it will most likely
// require that a replica fragment be the source data.
srcCluster := c
if action == ResizeJobActionAdd && c.ReplicaN > 1 {
if action == resizeJobActionAdd && c.ReplicaN > 1 {
srcCluster = NewCluster()
srcCluster.Nodes = Nodes(c.Nodes).Clone()
srcCluster.Hasher = c.Hasher
@ -740,7 +731,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
srcNodesByFrag := make(map[frag]string)
for nodeID, frags := range srcFrags {
// If a node is being removed, don't consider it as a source.
if action == ResizeJobActionRemove && nodeID == diffNodeID {
if action == resizeJobActionRemove && nodeID == diffNodeID {
continue
}
for _, frag := range frags {
@ -772,7 +763,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
}
src := &internal.ResizeSource{
Node: EncodeNode(c.nodeByID(srcNodeID)),
Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)),
Index: idx.Name(),
Field: frag.field,
View: frag.view,
@ -786,8 +777,8 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
return m, nil
}
// Partition returns the partition that a slice belongs to.
func (c *Cluster) Partition(index string, slice uint64) int {
// partition returns the partition that a slice belongs to.
func (c *Cluster) partition(index string, slice uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
@ -798,18 +789,18 @@ func (c *Cluster) Partition(index string, slice uint64) int {
return int(h.Sum64() % uint64(c.PartitionN))
}
// SliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) SliceNodes(index string, slice uint64) []*Node {
return c.PartitionNodes(c.Partition(index, slice))
// sliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) sliceNodes(index string, slice uint64) []*Node {
return c.partitionNodes(c.partition(index, slice))
}
// OwnsSlice returns true if a host owns a fragment.
func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID)
// ownsSlice returns true if a host owns a fragment.
func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID)
}
// PartitionNodes returns a list of nodes that own a partition.
func (c *Cluster) PartitionNodes(partitionID int) []*Node {
// partitionNodes returns a list of nodes that own a partition.
func (c *Cluster) partitionNodes(partitionID int) []*Node {
// Default replica count to between one and the number of nodes.
// The replica count can be zero if there are no nodes.
replicaN := c.ReplicaN
@ -831,27 +822,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node {
return nodes
}
// OwnsSlices finds the set of slices owned by the node per Index
func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 {
// containsSlices is like OwnsSlices, but it includes replicas.
func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
p := c.Partition(index, i)
// Determine primary owner node.
nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes))
if c.Nodes[nodeIndex].URI == uri {
slices = append(slices, i)
}
}
return slices
}
// ContainsSlices is like OwnsSlices, but it includes replicas.
func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
p := c.Partition(index, i)
p := c.partition(index, i)
// Determine the nodes for partition.
nodes := c.PartitionNodes(p)
nodes := c.partitionNodes(p)
for _, n := range nodes {
if n.ID == node.ID {
slices = append(slices, i)
@ -884,7 +861,7 @@ func (h *jmphasher) Hash(key uint64, n int) int {
return int(b)
}
func (c *Cluster) Open() error {
func (c *Cluster) open() error {
// Cluster always comes up in state STARTING until cluster membership is determined.
c.state = ClusterStateStarting
@ -896,7 +873,7 @@ func (c *Cluster) Open() error {
c.ID = c.Topology.ClusterID
// Only the coordinator needs to consider the .topology file.
if c.IsCoordinator() {
if c.isCoordinator() {
err := c.considerTopology()
if err != nil {
return fmt.Errorf("considerTopology: %v", err)
@ -904,7 +881,7 @@ func (c *Cluster) Open() error {
}
// Add the local node to the cluster.
err := c.AddNode(c.Node)
err := c.addNode(c.Node)
if err != nil {
return errors.Wrap(err, "adding local node")
}
@ -920,7 +897,7 @@ func (c *Cluster) Open() error {
}
// If not coordinator then wait for ClusterStatus from coordinator.
if !c.IsCoordinator() {
if !c.isCoordinator() {
// In the case where a node has been restarted and memberlist has
// not had enough time to determine the node went down/up, then
// the coorninator needs to be alerted that this node is back up
@ -945,7 +922,7 @@ func (c *Cluster) Open() error {
return nil
}
func (c *Cluster) Close() error {
func (c *Cluster) close() error {
// Notify goroutines of closing and wait for completion.
close(c.closing)
c.wg.Wait()
@ -962,14 +939,14 @@ func (c *Cluster) markAsJoined() {
}
func (c *Cluster) needTopologyAgreement() bool {
return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs())
return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
}
func (c *Cluster) haveTopologyAgreement() bool {
if c.Static {
return true
}
return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs())
return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
}
func (c *Cluster) allNodesReady() bool {
@ -999,10 +976,10 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
// channel, which is not consumed until the code below.
var eg errgroup.Group
eg.Go(func() error {
return j.Run()
return j.run()
})
// Wait for the ResizeJob to finish or be aborted.
// Wait for the resizeJob to finish or be aborted.
c.Logger.Printf("wait for jobResult")
jobResult := <-j.result
@ -1013,18 +990,18 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error {
c.Logger.Printf("received jobResult: %s", jobResult)
switch jobResult {
case ResizeJobStateDone:
if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil {
case resizeJobStateDone:
if err := c.completeCurrentJob(resizeJobStateDone); err != nil {
return errors.Wrap(err, "completing finished job")
}
// Add/remove uri to/from the cluster.
if j.action == ResizeJobActionRemove {
return c.RemoveNode(nodeAction.node)
} else if j.action == ResizeJobActionAdd {
return c.AddNode(nodeAction.node)
if j.action == resizeJobActionRemove {
return c.removeNode(nodeAction.node)
} else if j.action == resizeJobActionAdd {
return c.addNode(nodeAction.node)
}
case ResizeJobStateAborted:
if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil {
case resizeJobStateAborted:
if err := c.completeCurrentJob(resizeJobStateAborted); err != nil {
return errors.Wrap(err, "completing aborted job")
}
}
@ -1045,64 +1022,63 @@ func (c *Cluster) sendTo(node *Node, msg proto.Message) error {
return nil
}
// ListenForJoins handles cluster-resize events.
func (c *Cluster) ListenForJoins() {
c.wg.Add(1)
go func() { defer c.wg.Done(); c.listenForJoins() }()
}
// listenForJoins handles cluster-resize events.
func (c *Cluster) listenForJoins() {
// When a cluster starts, the state is STARTING.
// We first want to wait for at least one node to join.
// Then we want to clear out the joiningLeavingNodes queue (buffered channel).
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
c.wg.Add(1)
go func() {
defer c.wg.Done()
var setNormal bool
// When a cluster starts, the state is STARTING.
// We first want to wait for at least one node to join.
// Then we want to clear out the joiningLeavingNodes queue (buffered channel).
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
var setNormal bool
for {
for {
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
default:
}
// Only change state to NORMAL if we have successfully added at least one host.
if setNormal {
// Put the cluster back to state NORMAL and broadcast.
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
}
// Wait for a joining host or a close.
select {
case <-c.closing:
return
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
}
setNormal = true
continue
default:
}
// Only change state to NORMAL if we have successfully added at least one host.
if setNormal {
// Put the cluster back to state NORMAL and broadcast.
if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil {
c.Logger.Printf("setStateAndBroadcast error: err=%s", err)
}
}
// Wait for a joining host or a close.
select {
case <-c.closing:
return
case nodeAction := <-c.joiningLeavingNodes:
err := c.handleNodeAction(nodeAction)
if err != nil {
c.Logger.Printf("handleNodeAction error: err=%s", err)
continue
}
setNormal = true
continue
}
}
}()
}
// generateResizeJob creates a new ResizeJob based on the new node being
// added/removed. It also saves a reference to the ResizeJob in the `jobs` map
// generateResizeJob creates a new resizeJob based on the new node being
// added/removed. It also saves a reference to the resizeJob in the `jobs` map
// for future lookup by JobID.
func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) {
c.Logger.Printf("generateResizeJob: %v", nodeAction)
c.mu.Lock()
defer c.mu.Unlock()
@ -1111,7 +1087,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
if err != nil {
return nil, errors.Wrap(err, "generating job")
}
c.Logger.Printf("generated ResizeJob: %d", j.ID)
c.Logger.Printf("generated resizeJob: %d", j.ID)
// Save job in jobs map for future reference.
c.jobs[j.ID] = j
@ -1125,12 +1101,12 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) {
return j, nil
}
// generateResizeJobByAction returns a ResizeJob with instructions based on
// generateResizeJobByAction returns a resizeJob with instructions based on
// the difference between Cluster and a new Cluster with/without uri.
// Broadcaster is associated to the ResizeJob here for use in broadcasting
// Broadcaster is associated to the resizeJob here for use in broadcasting
// the resize instructions to other nodes in the cluster.
func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) {
j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action)
func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) {
j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action)
j.Broadcaster = c.Broadcaster
// toCluster is a clone of Cluster with the new node added/removed for comparison.
@ -1139,9 +1115,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
toCluster.Hasher = c.Hasher
toCluster.PartitionN = c.PartitionN
toCluster.ReplicaN = c.ReplicaN
if nodeAction.action == ResizeJobActionRemove {
if nodeAction.action == resizeJobActionRemove {
toCluster.removeNodeBasicSorted(nodeAction.node)
} else if nodeAction.action == ResizeJobActionAdd {
} else if nodeAction.action == resizeJobActionAdd {
toCluster.addNodeBasicSorted(nodeAction.node)
}
@ -1172,8 +1148,8 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
}
instr := &internal.ResizeInstruction{
JobID: j.ID,
Node: EncodeNode(toCluster.nodeByID(id)),
Coordinator: EncodeNode(c.CoordinatorNode()),
Node: EncodeNode(toCluster.unprotectedNodeByID(id)),
Coordinator: EncodeNode(c.coordinatorNode()),
Sources: sources,
Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node.
ClusterStatus: c.Status(),
@ -1184,28 +1160,28 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob,
return j, nil
}
// CompleteCurrentJob sets the state of the current ResizeJob
// completeCurrentJob sets the state of the current resizeJob
// then removes the pointer to currentJob.
func (c *Cluster) CompleteCurrentJob(state string) error {
func (c *Cluster) completeCurrentJob(state string) error {
c.mu.Lock()
defer c.mu.Unlock()
if !c.isCoordinator() {
if !c.unprotectedIsCoordinator() {
return ErrNodeNotCoordinator
}
if c.currentJob == nil {
return ErrResizeNotRunning
}
c.currentJob.SetState(state)
c.currentJob.setState(state)
c.currentJob = nil
return nil
}
// FollowResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// followResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) error {
c.Logger.Printf("follow resize instruction on %s", c.Node.ID)
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil {
if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil {
return errors.Wrap(err, "merging cluster status")
}
@ -1297,13 +1273,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
return nil
}
func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
func (c *Cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
j := c.Job(complete.JobID)
j := c.job(complete.JobID)
// Abort the job if an error exists in the complete object.
if complete.Error != "" {
j.result <- ResizeJobStateAborted
j.result <- resizeJobStateAborted
return errors.New(complete.Error)
}
@ -1311,29 +1287,27 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc
defer j.mu.Unlock()
if j.isComplete() {
return fmt.Errorf("ResizeJob %d is no longer running", j.ID)
return fmt.Errorf("resize job %d is no longer running", j.ID)
}
// Mark host complete.
j.IDs[complete.Node.ID] = true
if !j.nodesArePending() {
j.result <- ResizeJobStateDone
j.result <- resizeJobStateDone
}
return nil
}
// Job returns a ResizeJob by id.
func (c *Cluster) Job(id int64) *ResizeJob {
// job returns a resizeJob by id.
func (c *Cluster) job(id int64) *resizeJob {
c.mu.RLock()
defer c.mu.RUnlock()
return c.job(id)
return c.jobs[id]
}
func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] }
type ResizeJob struct {
type resizeJob struct {
ID int64
IDs map[string]bool
Instructions []*internal.ResizeInstruction
@ -1348,15 +1322,15 @@ type ResizeJob struct {
Logger Logger
}
// NewResizeJob returns a new instance of ResizeJob.
func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
// newResizeJob returns a new instance of resizeJob.
func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob {
// Build a map of uris to track their resize status.
// The value for a node will be set to true after that node
// has indicated that it has completed all resize instructions.
ids := make(map[string]bool)
if action == ResizeJobActionRemove {
if action == resizeJobActionRemove {
for _, n := range existingNodes {
// Exclude the removed node from the map.
if n.ID == node.ID {
@ -1364,7 +1338,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
}
ids[n.ID] = false
}
} else if action == ResizeJobActionAdd {
} else if action == resizeJobActionAdd {
for _, n := range existingNodes {
ids[n.ID] = false
}
@ -1372,7 +1346,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
ids[node.ID] = false
}
return &ResizeJob{
return &resizeJob{
ID: rand.Int63(),
IDs: ids,
action: action,
@ -1381,50 +1355,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob {
}
}
func (j *ResizeJob) State() string {
j.mu.RLock()
defer j.mu.RUnlock()
return j.state
}
func (j *ResizeJob) SetState(state string) {
func (j *resizeJob) setState(state string) {
j.mu.Lock()
j.setState(state)
if j.state == "" || j.state == resizeJobStateRunning {
j.state = state
}
j.mu.Unlock()
}
func (j *ResizeJob) setState(state string) {
if j.state == "" || j.state == ResizeJobStateRunning {
j.state = state
}
}
// Run distributes ResizeInstructions.
func (j *ResizeJob) Run() error {
j.Logger.Printf("run ResizeJob")
// run distributes ResizeInstructions.
func (j *resizeJob) run() error {
j.Logger.Printf("run resizeJob")
// Set job state to RUNNING.
j.SetState(ResizeJobStateRunning)
j.setState(resizeJobStateRunning)
// Job can be considered done in the case where it doesn't require any action.
if !j.nodesArePending() {
j.Logger.Printf("ResizeJob contains no pending tasks; mark as done")
j.result <- ResizeJobStateDone
j.Logger.Printf("resizeJob contains no pending tasks; mark as done")
j.result <- resizeJobStateDone
return nil
}
j.Logger.Printf("distribute tasks for ResizeJob")
j.Logger.Printf("distribute tasks for resizeJob")
err := j.distributeResizeInstructions()
if err != nil {
j.result <- ResizeJobStateAborted
j.result <- resizeJobStateAborted
return errors.Wrap(err, "distributing instructions")
}
return nil
}
// isComplete return true if the job is any one of several completion states.
func (j *ResizeJob) isComplete() bool {
func (j *resizeJob) isComplete() bool {
switch j.state {
case ResizeJobStateDone, ResizeJobStateAborted:
case resizeJobStateDone, resizeJobStateAborted:
return true
default:
return false
@ -1432,7 +1396,7 @@ func (j *ResizeJob) isComplete() bool {
}
// nodesArePending returns true if any node is still working on the resize.
func (j *ResizeJob) nodesArePending() bool {
func (j *resizeJob) nodesArePending() bool {
for _, complete := range j.IDs {
if !complete {
return true
@ -1441,9 +1405,9 @@ func (j *ResizeJob) nodesArePending() bool {
return false
}
func (j *ResizeJob) distributeResizeInstructions() error {
func (j *resizeJob) distributeResizeInstructions() error {
j.Logger.Printf("distributeResizeInstructions for job %d", j.ID)
// Loop through the ResizeInstructions in ResizeJob and send to each host.
// Loop through the ResizeInstructions in resizeJob and send to each host.
for _, instr := range j.Instructions {
// Because the node may not be in the cluster yet, create
// a dummy node object to use in the SendTo() method.
@ -1659,7 +1623,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error {
case NodeJoin:
c.Logger.Printf("received NodeJoin event: %v", e)
// Ignore the event if this is not the coordinator.
if !c.IsCoordinator() {
if !c.isCoordinator() {
return nil
}
return c.nodeJoin(e.Node)
@ -1681,7 +1645,7 @@ func (c *Cluster) nodeJoin(node *Node) error {
return errors.New(err)
}
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node for agreement")
}
@ -1711,13 +1675,13 @@ func (c *Cluster) nodeJoin(node *Node) error {
// If the cluster already contains the node, just send it the cluster status.
// This is useful in the case where a node is restarted or temporarily leaves
// the cluster.
if node := c.nodeByID(node.ID); node != nil {
if node := c.unprotectedNodeByID(node.ID); node != nil {
return c.sendTo(node, c.Status())
}
// If the holder does not yet contain data, go ahead and add the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
@ -1730,16 +1694,16 @@ func (c *Cluster) nodeJoin(node *Node) error {
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd}
c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd}
return nil
}
// NodeLeave initiates the removal of a node from the cluster.
func (c *Cluster) NodeLeave(node *Node) error {
// nodeLeave initiates the removal of a node from the cluster.
func (c *Cluster) nodeLeave(node *Node) error {
// Refuse the request if this is not the coordinator.
if !c.IsCoordinator() {
return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID)
if !c.isCoordinator() {
return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID)
}
if c.State() != ClusterStateNormal {
@ -1747,7 +1711,7 @@ func (c *Cluster) NodeLeave(node *Node) error {
}
// Ensure that node is in the cluster.
if c.nodeByID(node.ID) == nil {
if c.unprotectedNodeByID(node.ID) == nil {
return fmt.Errorf("Node is not a member of the cluster: %s", node.ID)
}
@ -1757,18 +1721,12 @@ func (c *Cluster) NodeLeave(node *Node) error {
}
// See if resize job can be generated
_, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove})
if err != nil {
if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil {
return errors.Wrap(err, "generating job")
}
return c.nodeLeave(node)
}
func (c *Cluster) nodeLeave(node *Node) error {
// Get the actual node in the local cluster.
n := c.nodeByID(node.ID)
n := c.unprotectedNodeByID(node.ID)
// Don't do anything else if the cluster doesn't contain the node.
if n == nil {
@ -1777,7 +1735,7 @@ func (c *Cluster) nodeLeave(node *Node) error {
// If the holder does not yet contain data, go ahead and remove the node.
if ok, err := c.Holder.HasData(); !ok && err == nil {
if err := c.RemoveNode(n); err != nil {
if err := c.removeNode(n); err != nil {
return errors.Wrap(err, "removing node")
}
return c.setStateAndBroadcast(ClusterStateNormal)
@ -1790,17 +1748,17 @@ func (c *Cluster) nodeLeave(node *Node) error {
if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil {
return errors.Wrap(err, "broadcasting state")
}
c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove}
c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove}
return nil
}
func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.Logger.Printf("merge cluster status: %v", cs)
// Ignore status updates from self (coordinator).
if c.isCoordinator() {
if c.unprotectedIsCoordinator() {
return nil
}
@ -1811,7 +1769,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
// Add all nodes from the coordinator.
for _, node := range officialNodes {
if err := c.AddNode(node); err != nil {
if err := c.addNode(node); err != nil {
return errors.Wrap(err, "adding node")
}
}
@ -1832,7 +1790,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error {
}
for _, nodeID := range nodeIDsToRemove {
if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil {
if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil {
return errors.Wrap(err, "removing node")
}
}

470
cluster_internal_test.go
View file

@ -15,11 +15,15 @@
package pilosa
import (
"bytes"
"io/ioutil"
"math/rand"
"reflect"
"strings"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/internal"
)
@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) {
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
action: resizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
action: resizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
actual := newResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) {
}
}
}
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.containsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
// NEXT: move this test to internal and unexport IsCoordinator
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.isCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.isCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.addNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.nodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.updateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.updateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

494
cluster_test.go
View file

@ -1,494 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := NewTestCluster(5)
slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

3
ctl/server.go
View file

@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
// Translation
flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")

138
executor.go
View file

@ -25,13 +25,13 @@ import (
"github.com/pkg/errors"
)
// DefaultField is the field used if one is not specified.
// defaultField is the field used if one is not specified.
const (
DefaultField = "general"
defaultField = "general"
// MinThreshold is the lowest count to use in a Top-N operation when
// defaultMinThreshold is the lowest count to use in a Top-N operation when
// looking for additional id/count pairs.
MinThreshold = 1
defaultMinThreshold = 1
columnLabel = "col"
rowLabel = "row"
@ -50,6 +50,9 @@ type Executor struct {
// Maximum number of SetBit() or ClearBit() commands per request.
MaxWritesPerRequest int
// Stores key/id translation data.
TranslateStore TranslateStore
}
// ExecutorOption is a functional option type for pilosa.Executor
@ -83,6 +86,11 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
return nil, ErrIndexRequired
}
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
// Verify that the number of writes do not exceed the maximum.
if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest {
return nil, ErrTooManyWrites
@ -93,6 +101,29 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
opt = &ExecOptions{}
}
// Translate query keys to ids, if necessary.
for i := range q.Calls {
if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
return nil, err
}
}
results, err := e.execute(ctx, index, q, slices, opt)
if err != nil {
return nil, err
}
// Translate response objects from ids to keys, if necessary.
for i := range results {
results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
if err != nil {
return nil, err
}
}
return results, nil
}
func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Don't bother calculating slices for query types that don't require it.
needsSlices := needsSlices(q.Calls)
@ -588,7 +619,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
// Set default field.
if field == "" {
field = DefaultField
field = defaultField
}
f := e.Holder.Fragment(index, field, ViewStandard, slice)
@ -597,7 +628,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
}
if minThreshold <= 0 {
minThreshold = MinThreshold
minThreshold = defaultMinThreshold
}
if tanimotoThreshold > 100 {
@ -646,7 +677,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
// Fetch field & row label based on argument.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
field = defaultField
}
f := e.Holder.Field(index, field)
if f == nil {
@ -700,7 +731,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Parse field, use default if unset.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
field = defaultField
}
// Retrieve column label.
@ -752,7 +783,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, field, view, slice)
if f == nil {
continue
@ -1002,7 +1033,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.ClearBit(view, rowID, colID, nil)
@ -1078,7 +1109,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetBit(view, rowID, colID, timestamp)
@ -1414,7 +1445,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m
loop:
for _, slice := range slices {
for _, node := range e.Cluster.SliceNodes(index, slice) {
for _, node := range e.Cluster.sliceNodes(index, slice) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
continue loop
@ -1444,7 +1475,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
if !opt.Remote {
nodes = Nodes(e.Cluster.Nodes).Clone()
} else {
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
@ -1559,6 +1590,78 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu
}
}
func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error {
// Translate column key.
if idx.Keys() {
if value := callArgString(c, "col"); value != "" {
ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value})
if err != nil {
return err
}
c.Args["col"] = ids[0]
}
}
// Translate row key, if field is specified & key exists.
if fieldName := callArgString(c, "field"); fieldName != "" {
field := idx.Field(fieldName)
if field.Keys() {
if value := callArgString(c, "row"); value != "" {
ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value})
if err != nil {
return err
}
c.Args["row"] = ids[0]
}
}
}
// Translate child calls.
for _, child := range c.Children {
if err := e.translateCall(index, idx, child); err != nil {
return err
}
}
return nil
}
func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) {
switch result := result.(type) {
case *Row:
if idx.Keys() {
other := &Row{Attrs: result.Attrs}
for _, segment := range result.Segments() {
for _, col := range segment.Columns() {
key, err := e.TranslateStore.TranslateColumnToString(index, col)
if err != nil {
return nil, err
}
other.Keys = append(other.Keys, key)
}
}
return other, nil
}
case []Pair:
if fieldName := callArgString(call, "field"); fieldName != "" {
field := idx.Field(fieldName)
if field.Keys() {
other := make([]Pair, len(result))
for i := range result {
key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID)
if err != nil {
return nil, err
}
other[i] = Pair{Key: key, Count: result[i].Count}
}
return other, nil
}
}
}
return result, nil
}
// errSliceUnavailable is a marker error if no nodes are available.
var errSliceUnavailable = errors.New("slice unavailable")
@ -1670,3 +1773,12 @@ func (vc *ValCount) Larger(other ValCount) ValCount {
Count: vc.Count,
}
}
func callArgString(call *pql.Call, key string) string {
value, ok := call.Args[key]
if !ok {
return ""
}
s, _ := value.(string)
return s
}

190
executor_test.go
View file

@ -22,6 +22,8 @@ import (
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
@ -38,7 +40,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
@ -87,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
@ -101,6 +103,35 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
})
t.Run("Keys", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
`SetBit(field=f, row="bar", col="foo")`+"\n"+
`SetBit(field=f, row="baz", col="foo")`+"\n"+
`SetBit(field=f, row="bar", col="bat")`+"\n"+
`SetBit(field=f, row="bbb", col="aaa")`+"\n",
), nil, nil); err != nil {
t.Fatal(err)
}
if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(results, []interface{}{
&pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}},
}, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" {
t.Fatal(diff)
}
})
}
// Ensure a difference query can be executed.
@ -113,7 +144,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
@ -127,7 +158,7 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "general", 10, 1)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil {
t.Fatalf("Empty Difference query should give error, but got %v", res)
}
@ -145,7 +176,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
@ -158,7 +189,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil {
t.Fatalf("Empty Intersect query should give error, but got %v", res)
}
@ -175,7 +206,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
@ -189,7 +220,7 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "general", 10, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
@ -208,7 +239,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
@ -224,7 +255,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
@ -240,7 +271,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
@ -284,7 +315,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Set bsiGroup values.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil {
@ -322,21 +353,21 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType {
t.Fatalf("unexpected error: %s", err)
}
@ -359,7 +390,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other bitmaps and fields to test isolation.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -383,36 +414,36 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Ensure a TopN() query can be executed.
func TestExecutor_Execute_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
t.Run("ID", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=1)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(field=f, row=10, col=0)
SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=other, row=0, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=1)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(field=f, row=10, col=0)
SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=other, row=0, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
t.Run("Standard", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
@ -422,6 +453,46 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("Keys", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row="foo", col="a")
SetBit(field=f, row="foo", col="b")
SetBit(field=f, row="foo", col="c")
SetBit(field=f, row="foo", col="d")
SetBit(field=f, row="foo", col="e")
SetBit(field=f, row="bar", col="a")
SetBit(field=f, row="bar", col="b")
SetBit(field=f, row="baz", col="b")
SetBit(field=other, row="foo", col="a")
`), nil, nil); err != nil {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(result, []interface{}{
[]pilosa.Pair{
{Key: "foo", Count: 5},
{Key: "bar", Count: 2},
},
}); diff != "" {
t.Fatal(diff)
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
@ -437,7 +508,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr.SetBit("i", "f", 1, SliceWidth)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -471,7 +542,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr.SetBit("i", "f", 4, 3*SliceWidth+1)
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -506,7 +577,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -530,7 +601,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -553,7 +624,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
@ -567,7 +638,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
func TestExecutor_Execute_MinMax(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -662,7 +733,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -733,7 +804,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -775,7 +846,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
func TestExecutor_Execute_Range(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
@ -955,7 +1026,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure a remote query can return a row.
func TestExecutor_Execute_Remote_Row(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1003,7 +1074,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1038,7 +1109,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
@ -1090,7 +1161,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
@ -1144,7 +1215,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
c := test.NewCluster(2)
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
@ -1213,7 +1284,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.MaxWritesPerRequest = 3
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
t.Fatalf("unexpected error: %s", err)
@ -1229,7 +1301,7 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
targetAttrs := map[string]interface{}{
"foo": "bar",
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// SetColumnAttrs call should exclude the field attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil)

38
field.go
View file

@ -33,10 +33,10 @@ import (
const (
DefaultFieldType = FieldTypeSet
DefaultCacheType = CacheTypeRanked
defaultCacheType = CacheTypeRanked
// Default ranked field cache
DefaultCacheSize = 50000
defaultCacheSize = 50000
)
// Field types.
@ -82,7 +82,7 @@ func OptFieldFieldOptions(o FieldOptions) FieldOption {
// NewField returns a new instance of field.
func NewField(path, index, name string, opts ...FieldOption) (*Field, error) {
err := ValidateName(name)
err := validateName(name)
if err != nil {
return nil, err
}
@ -101,8 +101,8 @@ func NewField(path, index, name string, opts ...FieldOption) (*Field, error) {
options: FieldOptions{
Type: DefaultFieldType,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
CacheType: defaultCacheType,
CacheSize: defaultCacheSize,
},
Logger: NopLogger,
@ -282,6 +282,7 @@ func (f *Field) loadMeta() error {
f.options.Min = pb.Min
f.options.Max = pb.Max
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
return nil
}
@ -317,6 +318,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Min = 0
f.options.Max = 0
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
case FieldTypeInt:
f.options.Type = opt.Type
f.options.CacheType = CacheTypeNone
@ -324,6 +326,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Min = opt.Min
f.options.Max = opt.Max
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
// Create new bsiGroup.
bsig := &bsiGroup{
@ -345,6 +348,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.CacheSize = 0
f.options.Min = 0
f.options.Max = 0
f.options.Keys = opt.Keys
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
@ -378,6 +382,13 @@ func (f *Field) Close() error {
return nil
}
// Keys returns true if the field uses string keys.
func (f *Field) Keys() bool {
f.mu.RLock()
defer f.mu.RUnlock()
return f.options.Keys
}
// bsiGroup returns a bsiGroup by name.
func (f *Field) bsiGroup(name string) *bsiGroup {
f.mu.RLock()
@ -645,7 +656,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) {
// SetBit sets a bit on a view within the field.
func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
if !isValidView(name) {
return false, ErrInvalidView
}
@ -668,7 +679,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
}
// If a timestamp is specified then set bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
@ -687,7 +698,7 @@ func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
// ClearBit clears a bit within the field.
func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
if !isValidView(name) {
return false, ErrInvalidView
}
@ -710,7 +721,7 @@ func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
}
// If a timestamp is specified then clear bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
@ -899,7 +910,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
if timestamp == nil {
standard = []string{ViewStandard}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
standard = viewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
@ -1038,6 +1049,7 @@ type FieldOptions struct {
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Keys bool `json:"keys,omitempty"`
}
// Validate ensures that FieldOption values are valid.
@ -1075,6 +1087,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
Min: o.Min,
Max: o.Max,
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
}
}
@ -1089,6 +1102,7 @@ func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions {
Min: options.Min,
Max: options.Max,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Keys: options.Keys,
}
}
@ -1233,8 +1247,8 @@ const (
CacheTypeNone = "none"
)
// IsValidCacheType returns true if v is a valid cache type.
func IsValidCacheType(v string) bool {
// isValidCacheType returns true if v is a valid cache type.
func isValidCacheType(v string) bool {
switch v {
case CacheTypeLRU, CacheTypeRanked, CacheTypeNone:
return true

40
fragment.go
View file

@ -25,7 +25,6 @@ import (
"hash"
"io"
"io/ioutil"
"net/http"
"os"
"sort"
"sync"
@ -48,22 +47,20 @@ const (
// SliceWidth is the number of column IDs in a slice.
SliceWidth = 1048576
// SnapshotExt is the file extension used for an in-process snapshot.
SnapshotExt = ".snapshotting"
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
// CopyExt is the file extension used for the temp file used while copying.
CopyExt = ".copying"
// copyExt is the file extension used for the temp file used while copying.
copyExt = ".copying"
// CacheExt is the file extension for persisted cache ids.
CacheExt = ".cache"
// cacheExt is the file extension for persisted cache ids.
cacheExt = ".cache"
// HashBlockSize is the number of rows in a merkle hash block.
HashBlockSize = 100
)
const (
// DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
DefaultFragmentMaxOpN = 2000
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a field and slice in an index.
@ -120,18 +117,18 @@ func NewFragment(path, index, field, view string, slice uint64) *Fragment {
field: field,
view: view,
slice: slice,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
CacheType: defaultCacheType,
CacheSize: defaultCacheSize,
Logger: NopLogger,
MaxOpN: DefaultFragmentMaxOpN,
MaxOpN: defaultFragmentMaxOpN,
stats: NopStatsClient,
}
}
// cachePath returns the path to the fragment's cache data.
func (f *Fragment) cachePath() string { return f.path + CacheExt }
func (f *Fragment) cachePath() string { return f.path + cacheExt }
// Open opens the underlying storage.
func (f *Fragment) Open() error {
@ -1432,7 +1429,7 @@ func (f *Fragment) snapshot() error {
defer track(start, completeMessage, f.stats, f.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
snapshotPath := f.path + snapshotExt
file, err := os.Create(snapshotPath)
if err != nil {
return fmt.Errorf("create snapshot file: %s", err)
@ -1636,7 +1633,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) {
func (f *Fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path + CopyExt
path := f.path + copyExt
file, err := os.Create(path)
if err != nil {
return errors.Wrap(err, "creating directory")
@ -1719,9 +1716,8 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
Closing <-chan struct{}
}
@ -1740,7 +1736,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) syncFragment() error {
// Determine replica set.
nodes := s.Cluster.SliceNodes(s.Fragment.index, s.Fragment.slice)
nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice)
if len(nodes) == 1 {
return nil
}
@ -1821,7 +1817,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var uris []*URI
var pairSets []pairSet
for _, node := range s.Cluster.SliceNodes(f.index, f.slice) {
for _, node := range s.Cluster.sliceNodes(f.index, f.slice) {
if s.Node.ID == node.ID {
continue
}

2
fragment_internal_test.go
View file

@ -1245,7 +1245,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string)
file.Close()
if cacheType == "" {
cacheType = DefaultCacheType
cacheType = defaultCacheType
}
f := NewFragment(file.Name(), index, field, view, slice)

21
gossip/gossip.go
View file

@ -18,6 +18,7 @@ import (
"fmt"
"io/ioutil"
"log"
"net"
"strconv"
"strings"
"sync"
@ -213,7 +214,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
conf.AdvertiseAddr = hostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
@ -580,3 +581,21 @@ type Config struct {
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}
// hostToIP converts host to an IP4 address based on net.LookupIP().
func hostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}

6
handler.go
View file

@ -2,7 +2,7 @@ package pilosa
import (
"encoding/json"
"net/http"
"net"
)
// QueryRequest represent a request to process a query.
@ -61,13 +61,13 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
}
type Handler interface {
http.Handler
Serve(ln net.Listener, closing <-chan struct{})
GetAPI() *API
}
type NopHandler struct{}
func (n *NopHandler) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {}
func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {}
func (n *NopHandler) GetAPI() *API {
return nil

33
holder.go
View file

@ -18,7 +18,6 @@ import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
@ -34,8 +33,8 @@ import (
)
const (
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
DefaultCacheFlushInterval = 1 * time.Minute
// defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
defaultCacheFlushInterval = 1 * time.Minute
// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
FileLimit = 262144 // (512^2)
@ -84,7 +83,7 @@ func NewHolder() *Holder {
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
CacheFlushInterval: defaultCacheFlushInterval,
Logger: NopLogger,
}
@ -112,7 +111,8 @@ func (h *Holder) Open() error {
}
for _, fi := range fis {
if !fi.IsDir() {
// Skip files or hidden directories.
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
continue
}
@ -339,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
return nil, errors.Wrap(err, "creating")
}
index.keys = opt.Keys
if err := index.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
} else if err := index.saveMeta(); err != nil {
return nil, errors.Wrap(err, "meta")
}
// Update options.
h.indexes[index.Name()] = index
return index, nil
@ -563,9 +566,8 @@ func (h *Holder) logStartup() error {
type HolderSyncer struct {
Holder *Holder
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
// Stats
Stats StatsClient
@ -619,7 +621,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) {
continue
}
@ -755,11 +757,10 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
}
if err := fs.syncFragment(); err != nil {
return errors.Wrap(err, "syncing fragment")
@ -799,7 +800,7 @@ func (c *HolderCleaner) CleanHolder() error {
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, field := range index.Fields() {

16
holder_test.go
View file

@ -362,7 +362,6 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
client := http.GetHTTPClient(nil)
httpClient := http.NewInternalClientFromURI(uri, client)
cluster.InternalClient = httpClient
cluster.RemoteClient = client
// Create a local holder.
hldr0 := test.MustOpenHolder()
@ -383,7 +382,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create fields on nodes.
@ -419,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Set up syncer.
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: http.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
Stats: pilosa.NopStatsClient,
}
if err := syncer.SyncHolder(); err != nil {
@ -456,7 +454,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
cluster := pilosa.NewTestCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
@ -465,7 +463,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0} {

39
http/client_test.go
View file

@ -87,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
{0, 6, 9, 15, 16, 20},
}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
for _, ownNum := range owns[i] {
if ownNum == num {
ownsNum = true
break
@ -141,10 +148,10 @@ func TestClient_MultiNode(t *testing.T) {
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
// Connect to each node to compare results.
client := make([]*test.Client, 3)
client[0] = test.MustNewClient(s[0].Host(), defaultClient)
client[1] = test.MustNewClient(s[1].Host(), defaultClient)
client[2] = test.MustNewClient(s[2].Host(), defaultClient)
client := make([]*Client, 3)
client[0] = MustNewClient(s[0].Host(), defaultClient)
client[1] = MustNewClient(s[1].Host(), defaultClient)
client[2] = MustNewClient(s[2].Host(), defaultClient)
topN := 4
queryRequest := &internal.QueryRequest{
@ -224,7 +231,7 @@ func TestClient_Import(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
c := MustNewClient(s.Host(), defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -269,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
c := MustNewClient(s.Host(), defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
@ -338,7 +345,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)
c := MustNewClient(s.Host(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)
if err != nil {
t.Fatal(err)
@ -355,3 +362,17 @@ func TestClient_FragmentBlocks(t *testing.T) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
}
}
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*http.InternalClient
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string, h *gohttp.Client) *Client {
c, err := http.NewInternalClient(host, h)
if err != nil {
panic(err)
}
return &Client{InternalClient: c}
}

64
http/handler.go
View file

@ -117,6 +117,18 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
return handler, nil
}
func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) {
server := &http.Server{Handler: h}
go func() {
<-closing
server.Close()
}()
err := server.Serve(ln)
if err != nil && err.Error() != "http: Server closed" {
h.Logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
@ -191,6 +203,8 @@ func NewRouter(handler *Handler) *mux.Router {
// For now we just do it for the most commonly used handler, /query
router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET")
router.Use(handler.queryArgValidator)
return router
}
@ -1166,6 +1180,56 @@ func (h *Handler) GetAPI() *pilosa.API {
type defaultClusterMessageResponse struct{}
// TranslateStoreBufferSize is the buffer size used for streaming data.
const TranslateStoreBufferSize = 65536
func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64)
rc, err := h.API.TranslateStore.Reader(r.Context(), offset)
if err == pilosa.ErrNotImplemented {
http.Error(w, err.Error(), http.StatusNotImplemented)
return
} else if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer rc.Close()
// Ensure reader is closed when the client disconnects.
go func() { <-r.Context().Done(); rc.Close() }()
// Flush header so client can continue.
w.WriteHeader(http.StatusOK)
if w, ok := w.(http.Flusher); ok {
w.Flush()
}
// Copy from reader to client until store or client disconnect.
buf := make([]byte, TranslateStoreBufferSize)
for {
// Read from store.
n, err := rc.Read(buf)
if err == io.EOF {
return
} else if err != nil {
h.Logger.Printf("http: translate store read error: %s", err)
return
} else if n == 0 {
continue
}
// Write to response & flush.
if _, err := w.Write(buf[:n]); err != nil {
h.Logger.Printf("http: translate store response write error: %s", err)
return
} else if w, ok := w.(http.Flusher); ok {
w.Flush()
}
}
}
type queryValidationSpec struct {
required []string
args map[string]struct{}

87
http/translator.go Normal file
View file

@ -0,0 +1,87 @@
package http
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"github.com/pilosa/pilosa"
)
// Ensure implementation implements inteface.
var _ pilosa.TranslateStore = (*TranslateStore)(nil)
// TranslateStore represents an implementation of TranslateStore that
// communicates over HTTP. This is used with the TranslateHandler.
type TranslateStore struct {
URL string
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(rawurl string) *TranslateStore {
return &TranslateStore{URL: rawurl}
}
// TranslateColumnsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateColumnToString is not currently implemented.
func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// TranslateRowsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateRowToString is not currently implemented.
func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// Reader returns a reader that can stream data from a remote store.
func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
// Generate remote URL.
u, err := url.Parse(s.URL)
if err != nil {
return nil, err
}
u.Path = "/translate/data"
u.RawQuery = (url.Values{
"offset": {strconv.FormatInt(off, 10)},
}).Encode()
// Connect a stream to the remote server.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
// Connect a stream to the remote server.
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err)
}
// Handle error codes or return body as stream.
switch resp.StatusCode {
case http.StatusOK:
return resp.Body, nil
case http.StatusNotImplemented:
resp.Body.Close()
return nil, pilosa.ErrNotImplemented
default:
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body))
}
}

134
http/translator_test.go Normal file
View file

@ -0,0 +1,134 @@
package http_test
import (
"context"
"io"
"io/ioutil"
"net/http/httptest"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/mock"
"github.com/pilosa/pilosa/test"
)
func TestTranslateStore_Reader(t *testing.T) {
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
t.Run("ServerDisconnect", func(t *testing.T) {
var mrc mock.ReadCloser
var readN int
mrc.ReadFunc = func(p []byte) (int, error) {
readN++
switch readN {
case 1:
copy(p, []byte("foo"))
return 3, nil
case 2:
copy(p, []byte("barbaz"))
return 6, nil
case 3:
return 0, io.EOF
default:
t.Fatal("unexpected read")
return 0, nil
}
}
var closeInvoked bool
mrc.CloseFunc = func() error {
closeInvoked = true
return nil
}
// Setup handler on test server.
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
if off != 100 {
t.Fatalf("unexpected off: %d", off)
}
return &mrc, nil
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
// Connect to server and stream all available data.
store := http.NewTranslateStore(s.URL)
rc, err := store.Reader(context.Background(), 100)
if err != nil {
t.Fatal(err)
} else if data, err := ioutil.ReadAll(rc); err != nil {
t.Fatal(err)
} else if string(data) != `foobarbaz` {
t.Fatalf("unexpected data: %q", data)
} else if err := rc.Close(); err != nil {
t.Fatal(err)
}
if !closeInvoked {
t.Fatal("expected server close")
}
})
// Ensure server closes store reader if client disconnects.
t.Run("ClientDisconnect", func(t *testing.T) {
// Setup mock so that Read() hangs.
done := make(chan struct{})
var mrc mock.ReadCloser
mrc.ReadFunc = func(p []byte) (int, error) {
<-done
return 0, io.EOF
}
var closeInvoked bool
mrc.CloseFunc = func() error {
closeInvoked = true
return nil
}
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return &mrc, nil
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
defer close(done)
// Connect to server and begin streaming.
ctx, cancel := context.WithCancel(context.Background())
store := http.NewTranslateStore(s.URL)
if _, err := store.Reader(ctx, 0); err != nil {
t.Fatal(err)
}
// Cancel the context and check if server is closed.
cancel()
time.Sleep(100 * time.Millisecond)
if !closeInvoked {
t.Fatal("expected server-side close")
}
})
})
// Ensure client is notified if the server doesn't support streaming replication.
t.Run("ErrNotImplemented", func(t *testing.T) {
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return nil, pilosa.ErrNotImplemented
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
_, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0)
if err != pilosa.ErrNotImplemented {
t.Fatalf("unexpected error: %s", err)
}
})
}

26
index.go
View file

@ -33,6 +33,7 @@ type Index struct {
mu sync.RWMutex
path string
name string
keys bool // use string keys
// Fields by name.
fields map[string]*Field
@ -53,7 +54,7 @@ type Index struct {
// NewIndex returns a new instance of Index.
func NewIndex(path, name string) (*Index, error) {
err := ValidateName(name)
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name }
// Path returns the path the index was initialized with.
func (i *Index) Path() string { return i.path }
// Keys returns true if the index uses string keys.
func (i *Index) Keys() bool { return i.keys }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
@ -164,18 +168,17 @@ func (i *Index) loadMeta() error {
}
// Copy metadata fields.
i.keys = pb.Keys
return nil
}
// NOTE: Until we introduce new attributes to store in the index .meta file,
// we don't need to actually write the file. The code related to index.options
// and the index meta file are left in place for future use.
/*
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{})
buf, err := proto.Marshal(&internal.IndexMeta{
Keys: i.keys,
})
if err != nil {
return errors.Wrap(err, "marshalling")
}
@ -187,7 +190,6 @@ func (i *Index) saveMeta() error {
return nil
}
*/
// Close closes the index and its fields.
func (i *Index) Close() error {
@ -295,7 +297,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, e
func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
if name == "" {
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
@ -407,11 +409,15 @@ func encodeIndex(d *Index) *internal.Index {
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct{}
type IndexOptions struct {
Keys bool `json:"keys"`
}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{}
return &internal.IndexMeta{
Keys: i.Keys,
}
}
// hasTime returns true if a contains a non-nil time.

215
inmem/translator.go Normal file
View file

@ -0,0 +1,215 @@
package inmem
import (
"context"
"io"
"sync"
"github.com/pilosa/pilosa"
)
// Ensure type implements interface.
var _ pilosa.TranslateStore = &TranslateStore{}
// TranslateStore is an in-memory storage engine for translating string-to-uint64 values.
type TranslateStore struct {
mu sync.RWMutex
cols map[string]*translateIndex
rows map[frameKey]*translateIndex
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore() *TranslateStore {
return &TranslateStore{
cols: make(map[string]*translateIndex),
rows: make(map[frameKey]*translateIndex),
}
}
// Reader returns an error because it is not supported by the inmem store.
func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
return nil, pilosa.ErrReplicationNotSupported
}
// TranslateColumnsToUint64 converts value to a uint64 id.
// If value does not have an associated id then one is created.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
ret := make([]uint64, len(values))
// Read value under read lock.
s.mu.RLock()
if idx := s.cols[index]; idx != nil {
var writeRequired bool
for i := range values {
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
}
ret[i] = v
}
if !writeRequired {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
// If any values not found then recheck and then add under a write lock.
s.mu.Lock()
defer s.mu.Unlock()
// Recheck if value was created between the read lock and write lock.
idx := s.cols[index]
if idx != nil {
var writeRequired bool
for i := range values {
if ret[i] != 0 {
continue
}
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
continue
}
ret[i] = v
}
if !writeRequired {
return ret, nil
}
}
// Create index map if it doesn't exists.
if idx == nil {
idx = newTranslateIndex()
s.cols[index] = idx
}
// Add new identifiers.
for i := range values {
if ret[i] != 0 {
continue
}
idx.seq++
v := idx.seq
ret[i] = v
idx.lookup[values[i]] = v
idx.reverse[v] = values[i]
}
return ret, nil
}
// TranslateColumnToString converts a uint64 id to its associated string value.
// If the id is not associated with a string value then a blank string is returned.
func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.cols[index]; idx != nil {
if ret, ok := idx.reverse[value]; ok {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
return "", nil
}
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
key := frameKey{index, frame}
ret := make([]uint64, len(values))
// Read value under read lock.
s.mu.RLock()
if idx := s.rows[key]; idx != nil {
var writeRequired bool
for i := range values {
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
}
ret[i] = v
}
if !writeRequired {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
// If any values not found then recheck and then add under a write lock.
s.mu.Lock()
defer s.mu.Unlock()
// Recheck if value was created between the read lock and write lock.
idx := s.rows[key]
if idx != nil {
var writeRequired bool
for i := range values {
if ret[i] != 0 {
continue
}
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
continue
}
ret[i] = v
}
if !writeRequired {
return ret, nil
}
}
// Create map if it doesn't exists.
if idx == nil {
idx = newTranslateIndex()
s.rows[key] = idx
}
// Add new identifiers.
for i := range values {
if ret[i] != 0 {
continue
}
idx.seq++
v := idx.seq
ret[i] = v
idx.lookup[values[i]] = v
idx.reverse[v] = values[i]
}
return ret, nil
}
func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.rows[frameKey{index, frame}]; idx != nil {
if ret, ok := idx.reverse[value]; ok {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
return "", nil
}
type frameKey struct {
index string
frame string
}
type translateIndex struct {
seq uint64
lookup map[string]uint64
reverse map[uint64]string
}
func newTranslateIndex() *translateIndex {
return &translateIndex{
lookup: make(map[string]uint64),
reverse: make(map[uint64]string),
}
}

132
inmem/translator_test.go Normal file
View file

@ -0,0 +1,132 @@
package inmem_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
"github.com/pilosa/pilosa/inmem"
)
func TestTranslateStore_TranslateColumn(t *testing.T) {
s := inmem.NewTranslateStore()
// First translation should start id at zero.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
}
func TestTranslateStore_TranslateRow(t *testing.T) {
s := inmem.NewTranslateStore()
// First translation should start id at zero.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different frame restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
}
func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) {
const batchSize = 1000
s := inmem.NewTranslateStore()
// Generate keys before benchmark begins
keySets := make([][]string, b.N/1000)
for i := range keySets {
keySets[i] = make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i)
}
}
b.ResetTimer()
for _, keySet := range keySets {
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) {
const batchSize = 1000
s := inmem.NewTranslateStore()
// Generate keys before benchmark begins
for i := 0; i < b.N; i += batchSize {
keySet := make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySet[j] = fmt.Sprintf("%08d%08d", jv, i)
}
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
// Generate random key access.
perm := rand.New(rand.NewSource(0)).Perm(b.N)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil {
b.Fatal(err)
}
}
}

349
internal/private.pb.go
View file

@ -1,6 +1,5 @@
// Code generated by protoc-gen-gogo.
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: private.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -61,6 +60,7 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type IndexMeta struct {
Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"`
}
func (m *IndexMeta) Reset() { *m = IndexMeta{} }
@ -68,6 +68,13 @@ func (m *IndexMeta) String() string { return proto.CompactTextString(
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
func (m *IndexMeta) GetKeys() bool {
if m != nil {
return m.Keys
}
return false
}
type FieldOptions struct {
Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
@ -75,6 +82,7 @@ type FieldOptions struct {
Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"`
Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"`
}
func (m *FieldOptions) Reset() { *m = FieldOptions{} }
@ -124,6 +132,13 @@ func (m *FieldOptions) GetTimeQuantum() string {
return ""
}
func (m *FieldOptions) GetKeys() bool {
if m != nil {
return m.Keys
}
return false
}
type ImportResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
}
@ -966,6 +981,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.Keys {
dAtA[i] = 0x18
i++
if m.Keys {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -1017,6 +1042,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Max))
}
if m.Keys {
dAtA[i] = 0x58
i++
if m.Keys {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -2105,24 +2140,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Private(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Private(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -2135,6 +2152,9 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
func (m *IndexMeta) Size() (n int) {
var l int
_ = l
if m.Keys {
n += 2
}
return n
}
@ -2162,6 +2182,9 @@ func (m *FieldOptions) Size() (n int) {
if m.Max != 0 {
n += 1 + sovPrivate(uint64(m.Max))
}
if m.Keys {
n += 2
}
return n
}
@ -2675,6 +2698,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Keys = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -2869,6 +2912,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error {
break
}
}
case 11:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Keys = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -3485,51 +3548,14 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
var keykey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
keykey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey := string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
if m.Standard == nil {
m.Standard = make(map[string]uint64)
}
if iNdEx < postIndex {
var valuekey uint64
var mapkey string
var mapvalue uint64
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
@ -3539,31 +3565,69 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
valuekey |= (uint64(b) & 0x7F) << shift
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
var mapvalue uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if iNdEx >= l {
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
} else {
iNdEx = entryPreIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
m.Standard[mapkey] = mapvalue
} else {
var mapvalue uint64
m.Standard[mapkey] = mapvalue
}
m.Standard[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
@ -6617,69 +6681,70 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1011 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35,
0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a,
0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x1b, 0x60, 0x10, 0x09, 0xe0, 0x49, 0x7a,
0xeb, 0xc1, 0xdd, 0xb5, 0xda, 0x51, 0x66, 0xc7, 0xc3, 0x8c, 0x27, 0xc9, 0xf6, 0xc0, 0x15, 0x2e,
0xdc, 0x11, 0x67, 0xfe, 0x18, 0x8e, 0xfc, 0x09, 0x28, 0xfc, 0x23, 0xc8, 0x9f, 0x3d, 0x8f, 0x64,
0x37, 0x4d, 0x15, 0x7a, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b,
0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61,
0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8a, 0x45, 0x32, 0xf9, 0x3e,
0x53, 0xb1, 0x4c, 0x0b, 0xf2, 0x01, 0xf4, 0x77, 0xf9, 0xf8, 0x85, 0x38, 0x9c, 0x65, 0x22, 0xf0,
0x86, 0xce, 0x66, 0x9f, 0x35, 0x8c, 0x5a, 0x1a, 0xc5, 0x2f, 0x45, 0xe0, 0x0f, 0x9d, 0xcd, 0x01,
0x6b, 0x18, 0x64, 0x08, 0xcb, 0x87, 0xf1, 0x54, 0xfc, 0x58, 0xf2, 0x54, 0x95, 0xd3, 0xa0, 0x83,
0xd6, 0x6d, 0x16, 0x21, 0xe0, 0xa3, 0xe3, 0x1e, 0x8a, 0xf0, 0x4c, 0x56, 0xc1, 0xdb, 0x8f, 0xd3,
0xa0, 0x3f, 0x74, 0x36, 0x3d, 0xa6, 0x8f, 0xc8, 0xe1, 0x67, 0x01, 0x58, 0x0e, 0x3f, 0xa3, 0x14,
0x56, 0xc2, 0x69, 0x26, 0x73, 0xc5, 0x44, 0x91, 0xc9, 0xb4, 0x40, 0xab, 0xbd, 0x3c, 0x0f, 0x1c,
0x74, 0xa4, 0x8f, 0xf4, 0x67, 0x58, 0xdd, 0x49, 0xe4, 0xf8, 0x78, 0xc4, 0x15, 0x67, 0xe2, 0xa7,
0x52, 0x14, 0x8a, 0xbc, 0x03, 0x1d, 0xac, 0xd5, 0xea, 0x19, 0x42, 0x73, 0xb1, 0xe6, 0xc0, 0x35,
0x5c, 0x24, 0x34, 0x17, 0xed, 0xb1, 0x6a, 0x9f, 0x19, 0x42, 0x73, 0xa3, 0x24, 0x1e, 0x9b, 0x6a,
0x7d, 0x66, 0x08, 0x5d, 0xc7, 0x93, 0x58, 0x9c, 0xda, 0x12, 0xf1, 0x4c, 0x43, 0x58, 0x6b, 0xc5,
0xb7, 0x69, 0xae, 0x43, 0x97, 0xc9, 0xd3, 0x70, 0x54, 0x04, 0xce, 0xd0, 0xdb, 0xf4, 0x99, 0xa5,
0xb0, 0x91, 0x32, 0x29, 0xa7, 0xa9, 0x16, 0xb9, 0x28, 0x6a, 0x18, 0xf4, 0x0e, 0x74, 0xb0, 0xab,
0xba, 0xca, 0xc6, 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xf4, 0xf7, 0xf9, 0x19, 0xa6, 0x51, 0x90, 0xc7,
0xd0, 0x8b, 0x14, 0x4f, 0x27, 0x3c, 0x9f, 0xa0, 0xd2, 0xf2, 0xc3, 0x8f, 0xb6, 0xaa, 0x41, 0x6f,
0xd5, 0x6a, 0x5b, 0x95, 0xce, 0x5e, 0xaa, 0xf2, 0x19, 0xab, 0x4d, 0xde, 0xff, 0x02, 0x06, 0x17,
0x44, 0x3a, 0xde, 0xb1, 0x98, 0x55, 0x5d, 0x3d, 0x16, 0x33, 0x5d, 0xff, 0x09, 0x4f, 0x4a, 0x81,
0xbd, 0xf2, 0x99, 0x21, 0x3e, 0x77, 0x3f, 0x75, 0xe8, 0x36, 0x90, 0xdd, 0x5c, 0x70, 0x25, 0x30,
0xc8, 0xbe, 0x28, 0x0a, 0xfe, 0x5c, 0x5c, 0xdd, 0x71, 0xd3, 0x45, 0xb7, 0xd5, 0x45, 0x7a, 0x1f,
0xc8, 0x48, 0x24, 0x42, 0x09, 0x8b, 0xc7, 0x57, 0x78, 0xa0, 0x51, 0x15, 0xed, 0x7a, 0x5d, 0x72,
0x0f, 0x7c, 0x0d, 0x6e, 0x0c, 0xb6, 0xfc, 0xf0, 0x76, 0xd3, 0x91, 0x1a, 0xf7, 0x0c, 0x15, 0x68,
0x52, 0x39, 0x45, 0x04, 0x5c, 0x5b, 0xc2, 0x02, 0xd0, 0xdc, 0xb7, 0xa1, 0x3c, 0x0c, 0xb5, 0xde,
0x84, 0x6a, 0x2f, 0x95, 0x8d, 0xb6, 0x5d, 0x95, 0x7b, 0xd3, 0x68, 0xf4, 0xa9, 0xe5, 0x6a, 0xfc,
0x1d, 0xf0, 0xa9, 0xb0, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, 0x2d,
0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc4, 0x94, 0x93, 0x8f,
0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, 0xfc,
0x17, 0xe6, 0x74, 0x0f, 0xba, 0x18, 0xbd, 0x08, 0xfc, 0xcb, 0x6e, 0x90, 0xcf, 0xac, 0x98, 0xee,
0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, 0x37, 0xb2, 0x50,
0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0x1f, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, 0xfa, 0x14, 0xfc,
0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, 0x88, 0xee, 0x6d,
0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, 0x57, 0xca, 0x7c,
0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, 0xed, 0x3e, 0x52,
0x5c, 0xd5, 0x80, 0x5f, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, 0x7a, 0xd5, 0x04,
0x91, 0xa0, 0xdf, 0x19, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, 0x0e, 0x06, 0xcc,
0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00,
0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, 0xf9, 0x05, 0xa9,
0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, 0xdf, 0x8e, 0x89,
0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, 0x18, 0x75, 0x7f,
0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, 0x61, 0x84, 0xf4,
0x09, 0xf4, 0x76, 0xa2, 0xf0, 0xeb, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, 0x8d, 0x3b, 0xff,
0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, 0xaf, 0xc4, 0x4d,
0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xdf, 0xa4, 0xd3, 0x3f,
0x5c, 0x58, 0x63, 0xa2, 0x88, 0x5f, 0x8a, 0x30, 0x2d, 0x54, 0x5e, 0x8e, 0xf5, 0x82, 0x6b, 0xfb,
0x6f, 0xe5, 0x33, 0xdb, 0x6d, 0x8f, 0x19, 0xe2, 0x75, 0xc0, 0x44, 0x1e, 0xc0, 0xf2, 0xe5, 0x05,
0x98, 0x57, 0x6d, 0xab, 0x90, 0x07, 0xb0, 0x14, 0xc9, 0x32, 0xd7, 0x48, 0x32, 0xeb, 0xdd, 0xba,
0x74, 0x4c, 0x66, 0x46, 0xcc, 0x2a, 0xb5, 0x16, 0x94, 0x3a, 0xaf, 0x86, 0x12, 0x79, 0x7c, 0x09,
0x4a, 0x41, 0x17, 0x0d, 0xde, 0x6b, 0x0c, 0x2e, 0x88, 0xd9, 0x45, 0x6d, 0xfa, 0xab, 0x03, 0xb7,
0xda, 0x29, 0xbc, 0xd6, 0x6e, 0xd4, 0x13, 0x71, 0x17, 0x4e, 0xc4, 0x5b, 0x34, 0x11, 0xbf, 0x99,
0x48, 0xf3, 0xca, 0x75, 0xda, 0xaf, 0xdc, 0x31, 0xdc, 0x99, 0x1b, 0xd3, 0xae, 0x9c, 0x66, 0x1a,
0x0f, 0xff, 0x63, 0x5c, 0xfa, 0xd6, 0xc8, 0x73, 0x3b, 0xa8, 0x3e, 0x33, 0x04, 0xfd, 0x0c, 0xde,
0x8d, 0x84, 0x6a, 0x0d, 0xa9, 0x42, 0xdb, 0x10, 0xbc, 0x03, 0x71, 0x7a, 0x45, 0xf9, 0x5a, 0x44,
0xbf, 0x84, 0xe0, 0x28, 0x9b, 0x70, 0x25, 0x6e, 0x64, 0xbd, 0x03, 0xbd, 0x43, 0x99, 0xc9, 0x44,
0x3e, 0x9f, 0x5d, 0xb3, 0xf5, 0x01, 0x2c, 0x99, 0x2b, 0xd2, 0x7c, 0x7c, 0xfa, 0xac, 0x22, 0xe9,
0x6d, 0x0d, 0xe8, 0x31, 0x4f, 0xc6, 0x65, 0xa2, 0xd3, 0xd0, 0x3f, 0xa0, 0x62, 0x67, 0xf5, 0xaf,
0xf3, 0x0d, 0xe7, 0xef, 0xf3, 0x0d, 0xe7, 0x9f, 0xf3, 0x0d, 0xe7, 0xf7, 0x7f, 0x37, 0xde, 0x7a,
0xd6, 0xc5, 0x1f, 0xed, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x97, 0xf0, 0x12, 0xfd, 0xe2,
0x0a, 0x00, 0x00,
// 1028 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35,
0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50,
0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0,
0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05,
0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4,
0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb,
0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0,
0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04,
0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7,
0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8,
0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05,
0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b,
0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74,
0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0,
0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79,
0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13,
0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10,
0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7,
0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac,
0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d,
0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d,
0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14,
0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a,
0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85,
0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27,
0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8,
0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22,
0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd,
0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18,
0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0,
0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac,
0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf,
0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c,
0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8,
0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33,
0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2,
0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c,
0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4,
0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32,
0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52,
0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09,
0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98,
0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00,
0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4,
0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89,
0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f,
0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa,
0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f,
0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc,
0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa,
0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d,
0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c,
0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b,
0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97,
0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70,
0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f,
0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66,
0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63,
0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf,
0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99,
0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a,
0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab,
0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1,
0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21,
0x0a, 0x0b, 0x00, 0x00,
}

2
internal/private.proto
View file

@ -3,6 +3,7 @@ syntax = "proto3";
package internal;
message IndexMeta {
bool Keys = 3;
}
message FieldOptions {
@ -12,6 +13,7 @@ message FieldOptions {
int64 Min = 9;
int64 Max = 10;
string TimeQuantum = 5;
bool Keys = 11;
}
message ImportResponse {

35
internal/public.pb.go
View file

@ -1,6 +1,5 @@
// Code generated by protoc-gen-gogo.
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: public.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import encoding_binary "encoding/binary"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
@ -799,7 +800,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue))))
encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
i += 8
}
return i, nil
}
@ -1235,24 +1237,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Public(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Public(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -2333,15 +2317,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if (iNdEx + 8) > l {
return io.ErrUnexpectedEOF
}
v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
v = uint64(dAtA[iNdEx-8])
v |= uint64(dAtA[iNdEx-7]) << 8
v |= uint64(dAtA[iNdEx-6]) << 16
v |= uint64(dAtA[iNdEx-5]) << 24
v |= uint64(dAtA[iNdEx-4]) << 32
v |= uint64(dAtA[iNdEx-3]) << 40
v |= uint64(dAtA[iNdEx-2]) << 48
v |= uint64(dAtA[iNdEx-1]) << 56
m.FloatValue = float64(math.Float64frombits(v))
default:
iNdEx = preIndex

190
internal/test/querygenerator.go Normal file
View file

@ -0,0 +1,190 @@
package test
import (
"fmt"
"strconv"
"strings"
"github.com/pilosa/pilosa/pql"
)
type Args map[string]interface{}
type Calls []*pql.Call
func PQL(calls ...*pql.Call) *pql.Query {
return &pql.Query{Calls: calls}
}
func Row(frame string, row int) *pql.Call {
return &pql.Call{
Name: "Row",
Args: Args{
"frame": frame,
"row": row,
},
}
}
func mutationArgs(args ...interface{}) Args {
rargs := make(Args)
for _, arg := range args {
switch v := arg.(type) {
case int:
rargs["column"] = v
case string:
if strings.Contains(v, "=") {
parts := strings.Split(v, "=")
rargs["frame"] = parts[0]
i, _ := strconv.ParseInt(parts[1], 10, 64)
rargs["value"] = i
} else {
rargs["timestamp"] = v
}
default:
fmt.Printf("wat %T!\n", v)
}
}
return rargs
}
func Set(args ...interface{}) *pql.Call {
return &pql.Call{Name: "Set", Args: mutationArgs(args...)}
}
func Clear(args ...interface{}) *pql.Call {
return &pql.Call{Name: "Clear", Args: mutationArgs(args...)}
}
func magic(args ...interface{}) (Args, Calls) {
var (
rargs Args
calls Calls
)
for _, arg := range args {
switch v := arg.(type) {
case Args:
rargs = v
case []*pql.Call:
calls = append(calls, v...)
default:
fmt.Printf("wat %T!\n", v)
}
}
return rargs, calls
}
func Count(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Count", Args: kvargs, Children: children}
}
func Union(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Union", Args: kvargs, Children: children}
}
func Intersect(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Intersect", Args: kvargs, Children: children}
}
func Difference(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Difference", Args: kvargs, Children: children}
}
func Xor(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Xor", Args: kvargs, Children: children}
}
func Between(frame string, min, max int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.BETWEEN,
"Value": []int{min, max},
},
}
}
func Lt(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.LT,
"Value": column,
},
}
}
func Lte(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.LTE,
"Value": column,
},
}
}
func Gt(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.GT,
"Value": column,
},
}
}
func Gte(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.GTE,
"Value": column,
},
}
}
func CompareCall(a, b *pql.Call) bool {
if a.Name != b.Name {
return false
}
for k, i := range a.Args {
switch v := i.(type) {
case []int:
bside := b.Args[k]
for j := range v {
if v[j] != bside.([]int)[j] {
return false
}
}
default:
if b.Args[k] != i {
return false
}
}
}
if len(a.Children) == len(b.Children) {
for i := range a.Children {
if !CompareCall(a.Children[i], b.Children[i]) {
return false
}
}
} else {
return false
}
return true
}
func Compare(a, b *pql.Query) bool {
for i := range a.Calls {
if !CompareCall(a.Calls[i], b.Calls[i]) {
return false
}
}
return true
}

219
internal/test/querygenerator_test.go Normal file
View file

@ -0,0 +1,219 @@
package test
import (
"testing"
"github.com/pilosa/pilosa/pql"
)
func TestPQL_Generator(t *testing.T) {
t.Run("pql.Query generator", func(t *testing.T) {
for _, u := range []struct {
pql string
calc *pql.Query
exp *pql.Query
}{
{
pql: "Union(Row(aaa=10),Row(bbb=9))",
calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Union",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Intersect(Row(aaa=10),Row(bbb=9))",
calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Intersect",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Difference(Row(aaa=10),Row(bbb=9))",
calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Difference",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Range(bbb > 20)",
calc: PQL(Gt("bbb", 20)),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Range",
Args: map[string]interface{}{
"Op": pql.GT,
"Value": 20,
},
},
},
},
},
{
pql: "Range(10 < bbb < 20)",
calc: PQL(Between("bbb", 10, 20)),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Range",
Args: map[string]interface{}{
"Op": pql.BETWEEN,
"Value": []int{10, 20},
},
},
},
},
},
{
pql: "Set(10, aaa=9)",
calc: PQL(Set(10, "aaa=9")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Set",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
},
},
},
},
},
{
pql: "Clear(10, aaa=10)",
calc: PQL(Clear(10, "aaa=9")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Clear",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
},
},
},
},
},
{
pql: `Set(10, aaa=10, "2017-03-02T03:00")`,
calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Set",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
"timestamp": "2017-03-02T03:00",
},
},
},
},
},
{
pql: `Count(Row(aaa=10))`,
calc: PQL(Count(Row("aaa", 10))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Count",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
},
},
},
},
},
{
pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))",
calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Intersect",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Union",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 12},
},
},
},
},
},
},
} {
if !Compare(u.calc, u.exp) {
t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql)
}
}
})
}

14
mock/mock.go Normal file
View file

@ -0,0 +1,14 @@
package mock
type ReadCloser struct {
ReadFunc func(p []byte) (int, error)
CloseFunc func() error
}
func (rc *ReadCloser) Read(p []byte) (int, error) {
return rc.ReadFunc(p)
}
func (rc *ReadCloser) Close() error {
return rc.CloseFunc()
}

38
mock/translator.go Normal file
View file

@ -0,0 +1,38 @@
package mock
import (
"context"
"io"
"github.com/pilosa/pilosa"
)
var _ pilosa.TranslateStore = (*TranslateStore)(nil)
type TranslateStore struct {
TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error)
TranslateColumnToStringFunc func(index string, values uint64) (string, error)
TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error)
TranslateRowToStringFunc func(index, frame string, values uint64) (string, error)
ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error)
}
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
return s.TranslateColumnsToUint64Func(index, values)
}
func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) {
return s.TranslateColumnToStringFunc(index, values)
}
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
return s.TranslateRowsToUint64Func(index, frame, values)
}
func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
return s.TranslateRowToStringFunc(index, frame, value)
}
func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
return s.ReaderFunc(ctx, off)
}

71
pilosa.go
View file

@ -16,9 +16,7 @@ package pilosa
import (
"errors"
"net"
"regexp"
"strings"
"github.com/pilosa/pilosa/internal"
)
@ -63,6 +61,8 @@ var (
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
ErrNotImplemented = errors.New("not implemented")
)
// ApiMethodNotAllowedError wraps an error value indicating that a particular
@ -85,6 +85,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
@ -108,26 +109,16 @@ func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// ValidateName ensures that the name is a valid format.
func ValidateName(name string) error {
// validateName ensures that the name is a valid format.
func validateName(name string) error {
if !nameRegexp.Match([]byte(name)) {
return ErrName
}
return nil
}
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
// stringSlicesAreEqual determines if two string slices are equal.
func stringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
@ -150,54 +141,6 @@ func StringSlicesAreEqual(a, b []string) bool {
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {
if strings.Contains(b, a) {
return true
}
}
return false
}
// HostToIP converts host to an IP4 address based on net.LookupIP().
func HostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {

43
pilosa_internal_test.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"testing"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if validateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if validateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}

48
pilosa_test.go
View file

@ -22,54 +22,6 @@ import (
_ "github.com/pilosa/pilosa/test"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if pilosa.ValidateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if pilosa.ValidateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}
func TestStringInSlice(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "localhost:10101"
if !pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s in %v", substr, list)
}
substr = "10101"
if pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s not in %v", substr, list)
}
}
func TestContainsSubstring(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "10101"
if !pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s contained in %v", substr, list)
}
substr = "4000"
if pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s in not contained in %v", substr, list)
}
}
func TestAddressWithDefaults(t *testing.T) {
tests := []struct {
addr string

33
pql/ast.go
View file

@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int {
return n
}
// HasKeys returns true if any call in the query uses keys and requires translation to ids.
func (q *Query) HasKeys() bool {
for _, call := range q.Calls {
if call.Args["col"] != nil {
if _, ok := call.Args["col"].(string); ok {
return true
}
}
if call.Args["row"] != nil {
if _, ok := call.Args["row"].(string); ok {
return true
}
}
}
return false
}
// String returns a string representation of the query.
func (q *Query) String() string {
a := make([]string, len(q.Calls))
@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
}
}
// StringArg is for reading the value at key from call.Args as a string. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. An error is returned if the value is not a string.
func (c *Call) StringArg(key string) (string, bool, error) {
val, ok := c.Args[key]
if !ok {
return "", false, nil
}
switch tval := val.(type) {
case string:
return tval, true, nil
default:
return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval)
}
}
// Keys returns a list of argument keys in sorted order.
func (c *Call) Keys() []string {
a := make([]string, 0, len(c.Args))

10
row.go
View file

@ -27,6 +27,9 @@ import (
type Row struct {
segments []RowSegment
// String keys translated to/from segment columns.
Keys []string
// Attributes associated with the row.
Attrs map[string]interface{}
}
@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) {
return s.ClearBit(i)
}
// Segments returns a list of all segments in the row.
func (r *Row) Segments() []RowSegment {
return r.segments
}
// segment returns a segment for a given slice.
// Returns nil if segment does not exist.
func (r *Row) segment(slice uint64) *RowSegment {
@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys,omitempty"`
}
o.Columns = r.Columns()
o.Keys = r.Keys
o.Attrs = r.Attrs
if o.Attrs == nil {

91
server.go
View file

@ -19,7 +19,6 @@ import (
"fmt"
"log"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
@ -53,17 +52,17 @@ type Server struct {
closing chan struct{}
// Internal
Holder *Holder
Cluster *Cluster
diagnostics *DiagnosticsCollector
executor *Executor
Holder *Holder
Cluster *Cluster
TranslateFile *TranslateFile
diagnostics *DiagnosticsCollector
executor *Executor
// External
handler Handler
Broadcaster Broadcaster
BroadcastReceiver BroadcastReceiver
Gossiper Gossiper
remoteClient *http.Client
systemInfo SystemInfo
gcNotifier GCNotifier
NewAttrStore func(string) AttrStore
@ -77,6 +76,8 @@ type Server struct {
diagnosticInterval time.Duration
maxWritesPerRequest int
primaryTranslateStore TranslateStore
defaultClient InternalClient
dataDir string
}
@ -162,15 +163,6 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption {
}
}
// TODO: Remove RemoteClient
func OptServerRemoteClient(c *http.Client) ServerOption {
return func(s *Server) error {
s.remoteClient = c
s.Cluster.RemoteClient = c
return nil
}
}
func OptServerInternalClient(c InternalClient) ServerOption {
return func(s *Server) error {
s.executor = NewExecutor(OptExecutorInternalQueryClient(c))
@ -180,6 +172,13 @@ func OptServerInternalClient(c InternalClient) ServerOption {
}
}
func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
return func(s *Server) error {
s.primaryTranslateStore = store
return nil
}
}
func OptServerStatsClient(sc StatsClient) ServerOption {
return func(s *Server) error {
s.Holder.Stats = sc
@ -252,6 +251,14 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.Cluster.Logger = s.logger
s.Cluster.Holder = s.Holder
// Initialize translation database.
s.TranslateFile = NewTranslateFile()
s.TranslateFile.Path = filepath.Join(path, "keys")
s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore
if err := s.TranslateFile.Open(); err != nil {
return nil, err
}
// update URI port with actual listener port. TODO this should probably be done outside of here.
if s.URI.Port() == 0 {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
@ -270,8 +277,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.Holder = s.Holder
s.executor.Node = node
s.executor.Cluster = s.Cluster
s.executor.TranslateStore = s.TranslateFile
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.handler.GetAPI().Executor = s.executor
s.handler.GetAPI().TranslateStore = s.TranslateFile
return s, nil
}
@ -313,18 +322,8 @@ func (s *Server) Open() error {
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
server := &http.Server{Handler: s.handler}
go func() {
<-s.closing
server.Close()
}()
err := server.Serve(s.ln)
if err != nil && err.Error() != "http: Server closed" {
s.logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// Serve handler.
go s.handler.Serve(s.ln, s.closing)
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
@ -332,7 +331,7 @@ func (s *Server) Open() error {
}
// Open Cluster management.
if err := s.Cluster.Open(); err != nil {
if err := s.Cluster.open(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
}
@ -340,7 +339,7 @@ func (s *Server) Open() error {
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.Cluster.SetNodeState(NodeStateReady); err != nil {
if err := s.Cluster.setNodeState(NodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
@ -349,7 +348,7 @@ func (s *Server) Open() error {
// the cluster without waiting for data to load on the coordinator. Before
// this starts, the joins are queued up in the Cluster.joiningLeavingNodes
// buffered channel.
s.Cluster.ListenForJoins()
s.Cluster.listenForJoins()
// Start background monitoring.
s.wg.Add(3)
@ -370,11 +369,14 @@ func (s *Server) Close() error {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.Close()
s.Cluster.close()
}
if s.Holder != nil {
s.Holder.Close()
}
if s.TranslateFile != nil {
s.TranslateFile.Close()
}
return nil
}
@ -424,7 +426,6 @@ func (s *Server) monitorAntiEntropy() {
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.remoteClient
syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
// Sync holders.
@ -493,26 +494,26 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
return err
}
case *internal.ClusterStatus:
err := s.Cluster.MergeClusterStatus(obj)
err := s.Cluster.mergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
err := s.Cluster.FollowResizeInstruction(obj)
err := s.Cluster.followResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
err := s.Cluster.MarkResizeInstructionComplete(obj)
err := s.Cluster.markResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.Cluster.SetCoordinator(DecodeNode(obj.New))
s.Cluster.setCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.Cluster.UpdateCoordinator(DecodeNode(obj.New))
s.Cluster.updateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State)
err := s.Cluster.receiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
@ -650,7 +651,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.NodeID)
@ -659,7 +660,7 @@ func (s *Server) monitorDiagnostics() {
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
@ -716,7 +717,7 @@ func (s *Server) monitorRuntime() {
// Record the number of go routines.
s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
// Open File handles.
if err == nil {
s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0)
@ -732,8 +733,8 @@ func (s *Server) monitorRuntime() {
}
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
// countOpenFiles on operating systems that support lsof.
func countOpenFiles() (int, error) {
switch runtime.GOOS {
case "darwin", "linux", "unix", "freebsd":
// -b option avoid kernel blocks
@ -747,9 +748,9 @@ func CountOpenFiles() (int, error) {
return len(lines), nil
case "windows":
// TODO: count open file handles on windows
return 0, errors.New("CountOpenFiles() on Windows is not supported")
return 0, errors.New("countOpenFiles() on Windows is not supported")
default:
return 0, errors.New("CountOpenFiles() on this OS is not supported")
return 0, errors.New("countOpenFiles() on this OS is not supported")
}
}

5
server/config.go
View file

@ -78,6 +78,11 @@ type Config struct {
// Gossip config is based around memberlist.Config.
Gossip gossip.Config `toml:"gossip"`
// Translation config supports translation store replication.
Translation struct {
PrimaryURL string `toml:"primary-url"`
}
AntiEntropy struct {
Interval toml.Duration `toml:"interval"`
} `toml:"anti-entropy"`

9
server/server.go
View file

@ -216,7 +216,12 @@ func (m *Command) SetupServer() error {
}
c := http.GetHTTPClient(TLSConfig)
api.RemoteClient = c
// Setup connection to primary store if this is a replica.
var primaryTranslateStore pilosa.TranslateStore
if m.Config.Translation.PrimaryURL != "" {
primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL)
}
m.Server, err = pilosa.NewServer(
pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
@ -235,8 +240,8 @@ func (m *Command) SetupServer() error {
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerRemoteClient(c),
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
)
return errors.Wrap(err, "new server")

16
server/server_test.go
View file

@ -21,7 +21,6 @@ import (
"io/ioutil"
"math/rand"
"reflect"
"runtime"
"sort"
"strings"
"testing"
@ -263,21 +262,6 @@ func tempMkdir(t *testing.T) string {
return dir
}
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported CountOpenFiles test on Windows.")
}
count, err := pilosa.CountOpenFiles()
if err != nil {
t.Errorf("CountOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("CountOpenFiles returned invalid value 0.")
}
}
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)

30
test/client.go → server_internal_test.go
View file

@ -12,24 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package test
package pilosa
import (
gohttp "net/http"
"github.com/pilosa/pilosa/http"
"runtime"
"testing"
)
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*http.InternalClient
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string, h *gohttp.Client) *Client {
c, err := http.NewInternalClient(host, h)
if err != nil {
panic(err)
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported countOpenFiles test on Windows.")
}
count, err := countOpenFiles()
if err != nil {
t.Errorf("countOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("countOpenFiles returned invalid value 0.")
}
return &Client{InternalClient: c}
}

14
server_test.go
View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (

10
statik/statik.go Normal file
View file

File diff suppressed because one or more lines are too long

6
stats.go
View file

@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient {
return &ExpvarStatsClient{
m: m,
tags: UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
}
}
@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error {
return nil
}
// UnionStringSlice returns a sorted set of tags which combine a & b.
func UnionStringSlice(a, b []string) []string {
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)

8
stats_test.go
View file

@ -95,7 +95,7 @@ func TestStatsCount_TopN(t *testing.T) {
// Execute query.
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "TopN" {
@ -124,7 +124,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
@ -154,7 +154,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
field := e.Holder.Field("d", "f")
if field == nil {
t.Fatal("field not found")
@ -184,7 +184,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx := e.Holder.Index("d")
if idx == nil {
t.Fatal("idex not found")

38
statsd/statsd.go
View file

@ -15,6 +15,7 @@
package statsd
import (
"sort"
"time"
"github.com/DataDog/datadog-go/statsd"
@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string {
func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
return &StatsClient{
client: c.client,
tags: pilosa.UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
}
}
@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) {
func (c *StatsClient) SetLogger(logger pilosa.Logger) {
c.logger = logger
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}

90
test/attr.go
View file

@ -1,90 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"io/ioutil"
"os"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

378
test/cluster.go
View file

@ -15,17 +15,10 @@
package test
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Hasher = newModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
ID: fmt.Sprintf("node%d", i),
URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)),
URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
@ -55,372 +48,19 @@ func NewCluster(n int) *pilosa.Cluster {
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// modHasher represents a simple, mod-based hashing.
type modHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
// newModHasher returns a new instance of ModHasher with n buckets.
func newModHasher() *modHasher { return &modHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
func (*modHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// NewURI is a test URI creator that intentionally swallows errors.
func NewURI(scheme, host string, port uint16) pilosa.URI {
// newURI is a test URI creator that intentionally swallows errors.
func newURI(scheme, host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewURIFromHostPort(host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// TestCluster represents a cluster of test nodes, each of which
// has a pilosa.Cluster.
type TestCluster struct {
Clusters []*pilosa.Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*pilosa.Node
}
func (t *TestCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateField(field, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Field(index, field)
if f == nil {
return fmt.Errorf("index/field does not exist: %s/%s", index, field)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *TestCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &pilosa.NodeEvent{
Event: pilosa.NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != pilosa.ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &pilosa.Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := pilosa.NewHolder()
h.Path = path
// cluster
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
c.Holder = h
c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the TestCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewTestCluster returns a new instance of test.Cluster.
func NewTestCluster(n int) *TestCluster {
tc := &TestCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *TestCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *TestCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(pilosa.NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *TestCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// TestCluster implements Broadcaster interface.
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *TestCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == pilosa.ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *TestCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Field(src.Index, src.Field)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := pilosa.DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

2
test/executor.go
View file

@ -20,6 +20,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/inmem"
"github.com/pilosa/pilosa/pql"
)
@ -42,6 +43,7 @@ func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
e := &Executor{Executor: executor}
e.Holder = holder
e.Cluster = cluster
e.TranslateStore = inmem.NewTranslateStore()
e.Node = cluster.Nodes[0]
return e
}

28
time.go
View file

@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) {
return q, nil
}
// ViewByTimeUnit returns the view name for time with a given quantum unit.
func ViewByTimeUnit(name string, t time.Time, unit rune) string {
// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
switch unit {
case 'Y':
return fmt.Sprintf("%s_%s", name, t.Format("2006"))
@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string {
}
}
// ViewsByTime returns a list of views for a given timestamp.
func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
// viewsByTime returns a list of views for a given timestamp.
func viewsByTime(name string, t time.Time, q TimeQuantum) []string {
a := make([]string, 0, len(q))
for _, unit := range q {
view := ViewByTimeUnit(name, t, unit)
view := viewByTimeUnit(name, t, unit)
if view == "" {
continue
}
@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
return a
}
// ViewsByTimeRange returns a list of views to traverse to query a time range.
func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
// viewsByTimeRange returns a list of views to traverse to query a time range.
func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
t := start
// Save flags for performance.
@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextDayGTE(t, end) {
break
} else if t.Hour() != 0 {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
continue
}
@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextMonthGTE(t, end) {
break
} else if t.Day() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
continue
}
@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextYearGTE(t, end) {
break
} else if t.Month() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
continue
}
@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
// Walk back down from largest units to smallest units.
for t.Before(end) {
if hasYear && nextYearGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'Y'))
results = append(results, viewByTimeUnit(name, t, 'Y'))
t = t.AddDate(1, 0, 0)
} else if hasMonth && nextMonthGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
} else if hasDay && nextDayGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
} else if hasHour {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
} else {
break

58
time_test.go → time_internal_test.go
View file

@ -12,28 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package pilosa
import (
"reflect"
"testing"
"time"
"github.com/pilosa/pilosa"
)
// Ensure string can be parsed into time quantum.
func TestParseTimeQuantum(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil {
if q, err := ParseTimeQuantum("YMDH"); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if q != pilosa.TimeQuantum("YMDH") {
} else if q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %#v", q)
}
})
t.Run("ErrInvalidTimeQuantum", func(t *testing.T) {
if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum {
if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
t.Fatalf("unexpected error: %s", err)
}
})
@ -44,22 +42,22 @@ func TestViewByTimeUnit(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("Y", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("M", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" {
if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("D", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("H", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
t.Fatalf("unexpected name: %s", s)
}
})
@ -70,14 +68,14 @@ func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH"))
a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) {
t.Fatalf("unexpected names: %+v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D"))
a := viewsByTime("F", ts, mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000102"}) {
t.Fatalf("unexpected names: %+v", a)
}
@ -87,82 +85,82 @@ func TestViewsByTime(t *testing.T) {
// Ensure sets of fields can be returned for a given time range.
func TestViewsByTimeRange(t *testing.T) {
t.Run("Y", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YM", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM"))
if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD"))
if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("M", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M"))
if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD"))
if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH"))
if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("DH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH"))
if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("H", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H"))
if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) {
t.Fatalf("unexpected fields: %#v", a)
}
})
}
// DefaultTimeLayout is the time layout used by the tests.
const DefaultTimeLayout = "2006-01-02 15:04"
// defaultTimeLayout is the time layout used by the tests.
const defaultTimeLayout = "2006-01-02 15:04"
// MustParseTime parses value using DefaultTimeLayout. Panic on error.
func MustParseTime(value string) time.Time {
v, err := time.Parse(DefaultTimeLayout, value)
// mustParseTime parses value using DefaultTimeLayout. Panic on error.
func mustParseTime(value string) time.Time {
v, err := time.Parse(defaultTimeLayout, value)
if err != nil {
panic(err)
}
return v
}
// MustParseTimeQuantum parses v into a time quantum. Panic on error.
func MustParseTimeQuantum(v string) pilosa.TimeQuantum {
q, err := pilosa.ParseTimeQuantum(v)
// mustParseTimeQuantum parses v into a time quantum. Panic on error.
func mustParseTimeQuantum(v string) TimeQuantum {
q, err := ParseTimeQuantum(v)
if err != nil {
panic(err)
}

1006
translate.go Normal file
View file

File diff suppressed because it is too large Load diff

565
translate_test.go Normal file
View file

@ -0,0 +1,565 @@
package pilosa_test
import (
"bufio"
"context"
"fmt"
"io/ioutil"
"math/rand"
"os"
"reflect"
"strconv"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/pilosa/pilosa"
)
func TestTranslateFile_TranslateColumn(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
// First translation should start id at zero.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
// Ensure that non-existent values return "".
if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil {
t.Fatal(err)
} else if value != "" {
t.Fatalf("unexpected value: %s", value)
}
// Reopen the store.
if err := s.Reopen(); err != nil {
t.Fatal(err)
}
// Ensure translation is still correct after reopen.
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure translation is still correct after reopen.
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{3}) {
t.Fatalf("unexpected id: %#v", ids)
}
}
func TestTranslateFile_TranslateColumn_Large(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
// Generate key/values.
for i := 0; i < 1000000; i += 1000 {
keys := make([]string, 1000)
for j := 0; j < 1000; j++ {
keys[j] = strconv.Itoa(i + j + 1)
}
ids, err := s.TranslateColumnsToUint64("IDX0", keys)
if err != nil {
t.Fatal(err)
}
for j, id := range ids {
if exp := uint64(i + j + 1); id != exp {
t.Fatalf("unexpected id: got=%d, exp=%d", id, exp)
}
}
}
// Verify values can be returned.
for i := 0; i < 1000000; i++ {
exp := strconv.Itoa(i + 1)
if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil {
t.Fatal(err)
} else if key != exp {
t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
}
}
// Reopen and re-verify.
if err := s.Reopen(); err != nil {
t.Fatal(err)
}
for i := 0; i < 1000000; i++ {
exp := strconv.Itoa(i + 1)
if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil {
t.Fatal(err)
} else if key != exp {
t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
}
}
}
func TestTranslateFile_TranslateRow(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
// First translation should start id at zero.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different frame restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
// Ensure that non-existent values return blank.
if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil {
t.Fatal(err)
} else if value != "" {
t.Fatalf("unexpected value: %s", value)
}
// Reopen the store.
if err := s.Reopen(); err != nil {
t.Fatal(err)
}
// Translation on a different frame restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
// Translate new row and increment sequence.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{3}) {
t.Fatalf("unexpected id: %#v", ids)
}
}
func TestTranslateFile_TranslateRow_Large(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
// Generate key/values.
for i := 0; i < 1000000; i += 1000 {
keys := make([]string, 1000)
for j := 0; j < 1000; j++ {
keys[j] = strconv.Itoa(i + j + 1)
}
ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys)
if err != nil {
t.Fatal(err)
}
for j, id := range ids {
if exp := uint64(i + j + 1); id != exp {
t.Fatalf("unexpected id: got=%d, exp=%d", id, exp)
}
}
}
// Verify values can be returned.
for i := 0; i < 1000000; i++ {
exp := strconv.Itoa(i + 1)
if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil {
t.Fatal(err)
} else if key != exp {
t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
}
}
// Reopen and re-verify.
if err := s.Reopen(); err != nil {
t.Fatal(err)
}
for i := 0; i < 1000000; i++ {
exp := strconv.Itoa(i + 1)
if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil {
t.Fatal(err)
} else if key != exp {
t.Fatalf("unexpected key: got=%q, exp=%q", key, exp)
}
}
}
func TestTranslateFile_Reader(t *testing.T) {
t.Run("NoOffset", func(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
t.Fatal(err)
}
rc, err := s.Reader(context.Background(), 0)
if err != nil {
t.Fatal(err)
}
brc := bufio.NewReader(rc)
defer rc.Close()
// Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b.
var entry pilosa.LogEntry
if n, err := entry.ReadFrom(brc); err != nil {
t.Fatal(err)
} else if n != 14 {
t.Fatalf("unexpected n: %d", n)
} else if diff := cmp.Diff(entry, pilosa.LogEntry{
Type: pilosa.LogEntryTypeInsertColumn,
Index: []byte("IDX0"),
IDs: []uint64{1},
Keys: [][]byte{[]byte("foo")},
Length: 13,
}); diff != "" {
t.Fatal(diff)
}
// Read second entry.
if _, err := entry.ReadFrom(brc); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(entry, pilosa.LogEntry{
Type: pilosa.LogEntryTypeInsertRow,
Index: []byte("IDX0"),
Frame: []byte("FRAME0"),
IDs: []uint64{1, 2},
Keys: [][]byte{[]byte("bar"), []byte("baz")},
Length: 24,
}); diff != "" {
t.Fatal(diff)
}
// Write new entry.
if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil {
t.Fatal(err)
}
// Read new entry.
if _, err := entry.ReadFrom(brc); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(entry, pilosa.LogEntry{
Type: pilosa.LogEntryTypeInsertColumn,
Index: []byte("IDX0"),
IDs: []uint64{2},
Keys: [][]byte{[]byte("xyz")},
Length: 13,
}); diff != "" {
t.Fatal(diff)
}
// Close reader and ensure it returns EOF.
if err := rc.Close(); err != nil {
t.Fatal(err)
} else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("WithOffset", func(t *testing.T) {
s := MustOpenTranslateFile()
defer s.MustClose()
if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
t.Fatal(err)
}
// Start offset after the first entry.
rc, err := s.Reader(context.Background(), 14)
if err != nil {
t.Fatal(err)
}
brc := bufio.NewReader(rc)
defer rc.Close()
// This should be the second entry.
var entry pilosa.LogEntry
if _, err := entry.ReadFrom(brc); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(entry, pilosa.LogEntry{
Type: pilosa.LogEntryTypeInsertRow,
Index: []byte("IDX0"),
Frame: []byte("FRAME0"),
IDs: []uint64{1, 2},
Keys: [][]byte{[]byte("bar"), []byte("baz")},
Length: 24,
}); diff != "" {
t.Fatal(diff)
}
})
}
func TestTranslateFile_PrimaryTranslateStore(t *testing.T) {
// Create a primary store that accepts writes.
primary := MustOpenTranslateFile()
defer primary.MustClose()
// Create a replica that accepts writes from primary.
replica := NewTranslateFile()
replica.PrimaryTranslateStore = primary
if err := replica.Open(); err != nil {
t.Fatal(err)
}
defer replica.MustClose()
// Write to the primary.
if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil {
t.Fatal(err)
}
// Attempt to read replica until writes appear.
if err := retryFor(2*time.Second, func() error {
// Verify that replica have received writes.
if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil {
return err
} else if value != "foo" {
return fmt.Errorf("unexpected column 1 value: %s", value)
}
if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil {
return err
} else if value != "bar" {
return fmt.Errorf("unexpected row 1 value: %s", value)
}
if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
return err
} else if value != "baz" {
return fmt.Errorf("unexpected row 2 value: %s", value)
}
return nil
}); err != nil {
t.Fatal(err)
}
// Disconnect primary store & write more values.
if err := primary.Reopen(); err != nil {
t.Fatal(err)
} else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil {
t.Fatal(err)
}
// Attempt to read replica until write appear.
if err := retryFor(2*time.Second, func() error {
if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil {
return err
} else if value != "baz" {
return fmt.Errorf("unexpected column 2 value: %s", value)
}
return nil
}); err != nil {
t.Fatal(err)
}
// Disconnect replica store & write more values.
if err := replica.Reopen(); err != nil {
t.Fatal(err)
} else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil {
t.Fatal(err)
}
// Attempt to read replica until write appear.
if err := retryFor(2*time.Second, func() error {
if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil {
return err
} else if value != "foobar" {
return fmt.Errorf("unexpected column 3 value: %s", value)
}
return nil
}); err != nil {
t.Fatal(err)
}
}
func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) {
const batchSize = 1000
s := MustOpenTranslateFile()
defer s.MustClose()
// Generate keys before benchmark begins
keySets := make([][]string, b.N/batchSize)
for i := range keySets {
keySets[i] = make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i)
}
}
b.ResetTimer()
for _, keySet := range keySets {
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) {
const batchSize = 1000
s := MustOpenTranslateFile()
defer s.MustClose()
// Generate keys before benchmark begins
for i := 0; i < b.N; i += batchSize {
keySet := make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySet[j] = fmt.Sprintf("%08d%08d", jv, i)
}
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
// Generate random key access.
perm := rand.New(rand.NewSource(0)).Perm(b.N)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil {
b.Fatal(err)
}
}
}
type TranslateFile struct {
*pilosa.TranslateFile
}
func NewTranslateFile() *TranslateFile {
f, err := ioutil.TempFile("", "")
if err != nil {
panic(err)
}
f.Close()
s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()}
s.Path = f.Name()
return s
}
func MustOpenTranslateFile() *TranslateFile {
s := NewTranslateFile()
if err := s.Open(); err != nil {
panic(err)
}
return s
}
func (s *TranslateFile) Close() error {
defer os.Remove(s.Path)
return s.TranslateFile.Close()
}
func (s *TranslateFile) MustClose() {
if err := s.Close(); err != nil {
panic(err)
}
}
// Reopen closes the store and opens a new instance of it for the same path.
func (s *TranslateFile) Reopen() error {
prev := s.TranslateFile
if err := s.TranslateFile.Close(); err != nil {
return err
}
s.TranslateFile = pilosa.NewTranslateFile()
s.Path = prev.Path
s.PrimaryTranslateStore = prev.PrimaryTranslateStore
if err := s.Open(); err != nil {
return err
}
return nil
}
// retryFor executes fn every 100ms until d time passes or until fn return nil.
func retryFor(d time.Duration, fn func() error) (err error) {
timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond)
defer timer.Stop()
defer ticker.Stop()
for {
if err = fn(); err == nil {
return nil
}
select {
case <-timer.C:
return err
case <-ticker.C:
}
}
}

16
utils_test.go → utils_internal_test.go
View file

@ -121,7 +121,7 @@ func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64,
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / SliceWidth
nodes := c0.SliceNodes(index, slice)
nodes := c0.sliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
@ -236,7 +236,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error)
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
c.addNode(n)
}
}
@ -273,13 +273,13 @@ func (t *ClusterCluster) SetState(state string) {
// Open opens all clusters in the test cluster.
func (t *ClusterCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
if err := c.open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(NodeStateReady); err != nil {
if err := c.setNodeState(NodeStateReady); err != nil {
return err
}
}
@ -288,7 +288,7 @@ func (t *ClusterCluster) Open() error {
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
t.Clusters[0].listenForJoins()
return nil
}
@ -296,7 +296,7 @@ func (t *ClusterCluster) Open() error {
// Close closes all clusters in the test cluster.
func (t *ClusterCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
err := c.close()
if err != nil {
return err
}
@ -310,7 +310,7 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
c.mergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == ClusterStateNormal && t.resizing {
@ -337,7 +337,7 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
go coord.markResizeInstructionComplete(obj)
}
return nil
}

6
view.go
View file

@ -34,8 +34,8 @@ const (
viewBSIGroupPrefix = "bsig_"
)
// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
// isValidView returns true if name is valid.
func isValidView(name string) bool {
return name == ViewStandard
}
@ -73,7 +73,7 @@ func NewView(path, index, field, name string, cacheSize uint32) *View {
name: name,
cacheSize: cacheSize,
cacheType: DefaultCacheType,
cacheType: defaultCacheType,
fragments: make(map[uint64]*Fragment),
broadcaster: NopBroadcaster,

2
view_internal_test.go
View file

@ -26,7 +26,7 @@ func mustOpenView(index, field, name string) *View {
panic(err)
}
v := NewView(path, index, field, name, DefaultCacheSize)
v := NewView(path, index, field, name, defaultCacheSize)
if err := v.open(); err != nil {
panic(err)
}