scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-05 08:10:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into changelog-1.0.0

Browse source
This commit is contained in:
Cody Soyland 2018-07-05 23:19:14 -05:00 • committed by GitHub
commit de8b6ffbb2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
84 changed files with 3128 additions and 2433 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
api.go
View file

@ -26,8 +26,6 @@ import (
"strings"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -38,22 +36,25 @@ type API struct {
holder *Holder
cluster *cluster
server *Server
Serializer Serializer
}
// APIOption is a functional option type for pilosa.API
type APIOption func(*API) error
// apiOption is a functional option type for pilosa.API
type apiOption func(*API) error
func OptAPIServer(s *Server) APIOption {
func OptAPIServer(s *Server) apiOption {
return func(a *API) error {
a.server = s
a.holder = s.holder
a.cluster = s.cluster
a.Serializer = s.serializer
return nil
}
}
// NewAPI returns a new API instance.
func NewAPI(opts ...APIOption) (*API, error) {
func NewAPI(opts ...apiOption) (*API, error) {
api := &API{}
for _, opt := range opts {
@ -89,7 +90,7 @@ func (api *API) validate(f apiMethod) error {
if _, ok := validAPIMethods[state][f]; ok {
return nil
}
return NewApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state))
return newApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state))
}
// Query parses a PQL query out of the request and executes it.
@ -185,12 +186,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
}
// Send the create index message to all nodes.
err = api.server.SendSync(
&internal.CreateIndexMessage{
&CreateIndexMessage{
Index: indexName,
Meta: options.Encode(),
Meta: &options,
})
if err != nil {
api.server.logger.Printf("problem sending CreateIndex message: %s", err)
return nil, errors.Wrap(err, "sending CreateIndex message")
}
api.holder.Stats.Count("createIndex", 1, 1.0)
@ -205,7 +205,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) {
index := api.holder.Index(indexName)
if index == nil {
return nil, NewNotFoundError(ErrIndexNotFound)
return nil, newNotFoundError(ErrIndexNotFound)
}
return index, nil
}
@ -224,7 +224,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
}
// Send the delete index message to all nodes.
err = api.server.SendSync(
&internal.DeleteIndexMessage{
&DeleteIndexMessage{
Index: indexName,
})
if err != nil {
@ -244,7 +244,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
}
// Apply functional options.
fo := fieldOptions{}
fo := FieldOptions{}
for _, opt := range opts {
err := opt(&fo)
if err != nil {
@ -255,7 +255,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
// Find index.
index := api.holder.Index(indexName)
if index == nil {
return nil, NewNotFoundError(ErrIndexNotFound)
return nil, newNotFoundError(ErrIndexNotFound)
}
// Create field.
@ -266,10 +266,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
// Send the create field message to all nodes.
err = api.server.SendSync(
&internal.CreateFieldMessage{
&CreateFieldMessage{
Index: indexName,
Field: fieldName,
Meta: fo.Encode(),
Meta: &fo,
})
if err != nil {
api.server.logger.Printf("problem sending CreateField message: %s", err)
@ -287,7 +287,7 @@ func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field,
field := api.holder.Field(indexName, fieldName)
if field == nil {
return nil, NewNotFoundError(ErrFieldNotFound)
return nil, newNotFoundError(ErrFieldNotFound)
}
return field, nil
}
@ -303,7 +303,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
// Find index.
index := api.holder.Index(indexName)
if index == nil {
return NewNotFoundError(ErrIndexNotFound)
return newNotFoundError(ErrIndexNotFound)
}
// Delete field from the index.
@ -313,7 +313,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
// Send the delete field message to all nodes.
err := api.server.SendSync(
&internal.DeleteFieldMessage{
&DeleteFieldMessage{
Index: indexName,
Field: fieldName,
})
@ -384,8 +384,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read body error"))
}
var req internal.BlockDataRequest
if err := proto.Unmarshal(reqBytes, &req); err != nil {
var req BlockDataRequest
if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error"))
}
@ -395,11 +395,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
return nil, ErrFragmentNotFound
}
var resp = internal.BlockDataResponse{}
var resp = BlockDataResponse{}
resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block))
// Encode response.
buf, err := proto.Marshal(&resp)
buf, err := api.Serializer.Marshal(&resp)
if err != nil {
return nil, errors.Wrap(err, "merge block response encoding error")
}
@ -442,11 +442,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error {
return errors.Wrap(err, "validating api method")
}
err := api.server.SendSync(&internal.RecalculateCaches{})
err := api.server.SendSync(&RecalculateCaches{})
if err != nil {
return errors.Wrap(err, "broacasting message")
}
api.holder.RecalculateCaches()
api.holder.recalculateCaches()
return nil
}
@ -463,23 +463,24 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
return errors.Wrap(err, "reading body")
}
// Marshal into request object.
pb, err := UnmarshalMessage(body)
typ := body[0]
msg := getMessage(typ)
err = api.server.serializer.Unmarshal(body[1:], msg)
if err != nil {
return errors.Wrap(err, "unmarshaling message")
return errors.Wrap(err, "deserializing cluster message")
}
// Forward the error message.
if err := api.server.receiveMessage(pb); err != nil {
if err := api.server.receiveMessage(msg); err != nil {
return errors.Wrap(err, "receiving message")
}
return nil
}
// Schema returns information about each index in Pilosa including which fields
// and views they contain.
func (api *API) Schema(ctx context.Context) []*Index {
return api.holder.Indexes()
// they contain.
func (api *API) Schema(ctx context.Context) []*IndexInfo {
return api.holder.limitedSchema()
}
// Views returns the views in the given field.
@ -521,7 +522,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
// Send the delete view message to all nodes.
err := api.server.SendSync(
&internal.DeleteViewMessage{
&DeleteViewMessage{
Index: indexName,
Field: fieldName,
View: viewName,
@ -542,7 +543,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At
// Retrieve index from holder.
index := api.holder.Index(indexName)
if index == nil {
return nil, NewNotFoundError(ErrIndexNotFound)
return nil, newNotFoundError(ErrIndexNotFound)
}
// Retrieve local blocks.
@ -553,7 +554,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At
// Read all attributes from all mismatched blocks.
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) {
for _, blockID := range attrBlocks(localBlocks).Diff(blocks) {
// Retrieve block data.
m, err := index.ColumnAttrStore().BlockData(blockID)
if err != nil {
@ -587,7 +588,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
// Read all attributes from all mismatched blocks.
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) {
for _, blockID := range attrBlocks(localBlocks).Diff(blocks) {
// Retrieve block data.
m, err := f.RowAttrStore().BlockData(blockID)
if err != nil {
@ -603,7 +604,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
}
// Import bulk imports data into a particular index,field,shard.
func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
func (api *API) Import(ctx context.Context, req *ImportRequest) error {
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -632,7 +633,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
}
// ImportValue bulk imports values into a particular field.
func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error {
func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error {
if err := api.validate(apiImportValue); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -642,7 +643,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = field.ImportValue(req.ColumnIDs, req.Values)
err = field.importValue(req.ColumnIDs, req.Values)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
@ -684,7 +685,7 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I
index := api.holder.Index(indexName)
if index == nil {
api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error())
return nil, nil, NewNotFoundError(ErrIndexNotFound)
return nil, nil, newNotFoundError(ErrIndexNotFound)
}
// Retrieve field.
@ -716,8 +717,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
// Send the set-coordinator message to new node.
err = api.server.SendTo(
newNode,
&internal.SetCoordinatorMessage{
New: EncodeNode(newNode),
&SetCoordinatorMessage{
New: newNode,
})
if err != nil {
return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err)

6
attr.go
View file

@ -82,12 +82,12 @@ type AttrBlock struct {
Checksum []byte `json:"checksum"`
}
// AttrBlocks represents a list of blocks.
type AttrBlocks []AttrBlock
// attrBlocks represents a list of blocks.
type attrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.

50
boltdb/attrstore.go
View file

@ -30,17 +30,17 @@ import (
"github.com/pkg/errors"
)
// AttrBlockSize is the size of attribute blocks for anti-entropy.
const AttrBlockSize = 100
// attrBlockSize is the size of attribute blocks for anti-entropy.
const attrBlockSize = 100
// AttrCache represents a cache for attributes.
type AttrCache struct {
// attrCache represents a cache for attributes.
type attrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *AttrCache) Get(id uint64) map[string]interface{} {
func (c *attrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
@ -57,40 +57,40 @@ func (c *AttrCache) Get(id uint64) map[string]interface{} {
}
// Set updates the cached attributes for a given id.
func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
func (c *attrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// AttrStore represents a storage layer for attributes.
type AttrStore struct {
// attrStore represents a storage layer for attributes.
type attrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *AttrCache
attrCache *attrCache
}
// NewAttrCache returns a new instance of AttrCache.
func NewAttrCache() *AttrCache {
return &AttrCache{
// newAttrCache returns a new instance of AttrCache.
func newAttrCache() *attrCache {
return &attrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &AttrStore{
return &attrStore{
path: path,
attrCache: NewAttrCache(),
attrCache: newAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *AttrStore) Path() string { return s.path }
func (s *attrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *AttrStore) Open() error {
func (s *attrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
@ -112,7 +112,7 @@ func (s *AttrStore) Open() error {
}
// Close closes the store.
func (s *AttrStore) Close() error {
func (s *attrStore) Close() error {
if s.db != nil {
s.db.Close()
}
@ -120,7 +120,7 @@ func (s *AttrStore) Close() error {
}
// Attrs returns a set of attributes by ID.
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
@ -147,7 +147,7 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
}
// SetAttrs sets attribute values for a given ID.
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
@ -184,7 +184,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
@ -220,7 +220,7 @@ func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
}
// Blocks returns a list of all blocks in the store.
func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) {
tx, err := s.db.Begin(false)
if err != nil {
return nil, errors.Wrap(err, "starting transaction")
@ -228,7 +228,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
defer tx.Rollback()
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize)
// Iterate over each block.
var blocks []pilosa.AttrBlock
@ -251,7 +251,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) {
}
// BlockData returns all data for a single block.
func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
m := make(map[uint64]map[string]interface{})
// Start read-only transaction.
@ -262,8 +262,8 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
defer tx.Rollback()
// Move to the start of the block.
min := u64tob(uint64(i) * AttrBlockSize)
max := u64tob(uint64(i+1) * AttrBlockSize)
min := u64tob(uint64(i) * attrBlockSize)
max := u64tob(uint64(i+1) * attrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.

155
broadcast.go
View file

@ -16,20 +16,27 @@ package pilosa
import (
"fmt"
"reflect"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
// Serializer is an interface for serializing pilosa types to bytes and back.
type Serializer interface {
Marshal(Message) ([]byte, error)
Unmarshal([]byte, Message) error
}
// broadcaster is an interface for broadcasting messages.
type broadcaster interface {
SendSync(pb proto.Message) error
SendAsync(pb proto.Message) error
SendTo(to *Node, pb proto.Message) error
SendSync(Message) error
SendAsync(Message) error
SendTo(*Node, Message) error
}
// Message is the interface implemented by all core pilosa types which can be serialized to messages.
// TODO add at least a single "isMessage()" method.
type Message interface{}
func init() {
NopBroadcaster = &nopBroadcaster{}
}
@ -40,13 +47,13 @@ var NopBroadcaster broadcaster
type nopBroadcaster struct{}
// SendSync A no-op implementation of Broadcaster SendSync method.
func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil }
func (nopBroadcaster) SendSync(Message) error { return nil }
// SendAsync A no-op implementation of Broadcaster SendAsync method.
func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil }
func (nopBroadcaster) SendAsync(Message) error { return nil }
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil }
func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
// Broadcast message types.
const (
@ -68,95 +75,91 @@ const (
messageTypeNodeStatus
)
// MarshalMessage encodes the protobuf message into a byte slice.
func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateShardMessage:
typ = messageTypeCreateShard
case *internal.CreateIndexMessage:
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = messageTypeDeleteIndex
case *internal.CreateFieldMessage:
typ = messageTypeCreateField
case *internal.DeleteFieldMessage:
typ = messageTypeDeleteField
case *internal.CreateViewMessage:
typ = messageTypeCreateView
case *internal.DeleteViewMessage:
typ = messageTypeDeleteView
case *internal.ClusterStatus:
typ = messageTypeClusterStatus
case *internal.ResizeInstruction:
typ = messageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = messageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = messageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = messageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = messageTypeNodeState
case *internal.RecalculateCaches:
typ = messageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = messageTypeNodeEvent
case *internal.NodeStatus:
typ = messageTypeNodeStatus
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
buf, err := proto.Marshal(m)
// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
// type info which is used by the internal messaging stuff.
func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
typ := getMessageType(m)
buf, err := s.Marshal(m)
if err != nil {
return nil, errors.Wrap(err, "marshalling")
return nil, errors.Wrap(err, "marshaling")
}
return append([]byte{typ}, buf...), nil
}
// UnmarshalMessage decodes the byte slice into a protobuf message.
func UnmarshalMessage(buf []byte) (proto.Message, error) {
typ, buf := buf[0], buf[1:]
var m proto.Message
func getMessage(typ byte) Message {
switch typ {
case messageTypeCreateShard:
m = &internal.CreateShardMessage{}
return &CreateShardMessage{}
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
return &CreateIndexMessage{}
case messageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
return &DeleteIndexMessage{}
case messageTypeCreateField:
m = &internal.CreateFieldMessage{}
return &CreateFieldMessage{}
case messageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
return &DeleteFieldMessage{}
case messageTypeCreateView:
m = &internal.CreateViewMessage{}
return &CreateViewMessage{}
case messageTypeDeleteView:
m = &internal.DeleteViewMessage{}
return &DeleteViewMessage{}
case messageTypeClusterStatus:
m = &internal.ClusterStatus{}
return &ClusterStatus{}
case messageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
return &ResizeInstruction{}
case messageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
return &ResizeInstructionComplete{}
case messageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
return &SetCoordinatorMessage{}
case messageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
return &UpdateCoordinatorMessage{}
case messageTypeNodeState:
m = &internal.NodeStateMessage{}
return &NodeStateMessage{}
case messageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
return &RecalculateCaches{}
case messageTypeNodeEvent:
m = &internal.NodeEventMessage{}
return &NodeEvent{}
case messageTypeNodeStatus:
m = &internal.NodeStatus{}
return &NodeStatus{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
panic(fmt.Sprintf("unknown message type %d", typ))
}
}
func getMessageType(m Message) byte {
switch m.(type) {
case *CreateShardMessage:
return messageTypeCreateShard
case *CreateIndexMessage:
return messageTypeCreateIndex
case *DeleteIndexMessage:
return messageTypeDeleteIndex
case *CreateFieldMessage:
return messageTypeCreateField
case *DeleteFieldMessage:
return messageTypeDeleteField
case *CreateViewMessage:
return messageTypeCreateView
case *DeleteViewMessage:
return messageTypeDeleteView
case *ClusterStatus:
return messageTypeClusterStatus
case *ResizeInstruction:
return messageTypeResizeInstruction
case *ResizeInstructionComplete:
return messageTypeResizeInstructionComplete
case *SetCoordinatorMessage:
return messageTypeSetCoordinator
case *UpdateCoordinatorMessage:
return messageTypeUpdateCoordinator
case *NodeStateMessage:
return messageTypeNodeState
case *RecalculateCaches:
return messageTypeRecalculateCaches
case *NodeEvent:
return messageTypeNodeEvent
case *NodeStatus:
return messageTypeNodeStatus
default:
panic(fmt.Sprintf("don't have type for message %#v", m))
}
if err := proto.Unmarshal(buf, m); err != nil {
return nil, errors.Wrap(err, "unmarshalling")
}
return m, nil
}

51
broadcast_test.go
View file

@ -1,51 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"reflect"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// Ensure a message can be marshaled and unmarshaled.
func TestMessage_Marshal(t *testing.T) {
testMessageMarshal(t, &internal.CreateShardMessage{
Index: "i",
Shard: 8,
})
testMessageMarshal(t, &internal.DeleteIndexMessage{
Index: "i",
})
}
func testMessageMarshal(t *testing.T, m proto.Message) {
marshalled, err := pilosa.MarshalMessage(m)
if err != nil {
t.Fatal(err)
}
unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(unmarshalled, m) {
t.Fatalf("unexpected message marshalling: %s", unmarshalled)
}
}

33
cache.go
View file

@ -22,7 +22,6 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/lru"
)
@ -318,22 +317,6 @@ type Pair struct {
Count uint64 `json:"count"`
}
func encodePair(p Pair) *internal.Pair {
return &internal.Pair{
ID: p.ID,
Key: p.Key,
Count: p.Count,
}
}
func decodePair(pb *internal.Pair) Pair {
return Pair{
ID: pb.ID,
Key: pb.Key,
Count: pb.Count,
}
}
// Pairs is a sortable slice of Pair objects.
type Pairs []Pair
@ -409,22 +392,6 @@ func (p Pairs) String() string {
return buf.String()
}
func EncodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])
}
return other
}
func decodePairs(a []*internal.Pair) []Pair {
other := make([]Pair, len(a))
for i := range a {
other[i] = decodePair(a[i])
}
return other
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

67
client.go
View file

@ -3,9 +3,6 @@ package pilosa
import (
"context"
"io"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// Bit represents the intersection of a row and a column. It can be specifed by
@ -36,8 +33,8 @@ type InternalClient interface {
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@ -49,88 +46,88 @@ type InternalClient interface {
BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
}
//===============
type InternalQueryClient interface {
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
}
type NopInternalQueryClient struct{}
type nopInternalQueryClient struct{}
func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func NewNopInternalQueryClient() *NopInternalQueryClient {
return &NopInternalQueryClient{}
func newNopInternalQueryClient() *nopInternalQueryClient {
return &nopInternalQueryClient{}
}
var _ InternalQueryClient = NewNopInternalQueryClient()
var _ InternalQueryClient = newNopInternalQueryClient()
//===============
type NopInternalClient struct{}
type nopInternalClient struct{}
func NewNopInternalClient() NopInternalClient {
return NopInternalClient{}
func newNopInternalClient() nopInternalClient {
return nopInternalClient{}
}
var _ InternalClient = NewNopInternalClient()
var _ InternalClient = newNopInternalClient()
func (n NopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
return nil, nil
}
func (n NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
return nil, nil
}
func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {
func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {
return nil
}
func (n NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error {
func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error {
return nil
}
func (n NopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
return nil
}
func (n NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n NopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error {
func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error {
return nil
}
func (n NopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
return nil
}
func (n NopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) {
func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) {
func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error {
func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
return nil
}
func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {
return nil, nil
}

331
cluster.go
View file

@ -36,8 +36,8 @@ import (
)
const (
// DefaultPartitionN is the default number of partitions in a cluster.
DefaultPartitionN = 256
// defaultPartitionN is the default number of partitions in a cluster.
defaultPartitionN = 256
// ClusterState represents the state returned in the /status endpoint.
ClusterStateStarting = "STARTING"
@ -45,7 +45,7 @@ const (
ClusterStateResizing = "RESIZING"
// NodeState represents the state of a node during startup.
NodeStateReady = "READY"
nodeStateReady = "READY"
// resizeJob states.
resizeJobStateRunning = "RUNNING"
@ -68,52 +68,6 @@ func (n Node) String() string {
return fmt.Sprintf("Node: %s", n.ID)
}
// EncodeNodes converts a slice of Nodes into its internal representation.
func EncodeNodes(a []*Node) []*internal.Node {
other := make([]*internal.Node, len(a))
for i := range a {
other[i] = EncodeNode(a[i])
}
return other
}
// EncodeNode converts a Node into its internal representation.
func EncodeNode(n *Node) *internal.Node {
return &internal.Node{
ID: n.ID,
URI: n.URI.Encode(),
IsCoordinator: n.IsCoordinator,
}
}
// DecodeNodes converts a proto message into a slice of Nodes.
func DecodeNodes(a []*internal.Node) []*Node {
if len(a) == 0 {
return nil
}
other := make([]*Node, len(a))
for i := range a {
other[i] = DecodeNode(a[i])
}
return other
}
// DecodeNode converts a proto message into a Node.
func DecodeNode(node *internal.Node) *Node {
return &Node{
ID: node.ID,
URI: decodeURI(node.URI),
IsCoordinator: node.IsCoordinator,
}
}
func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent {
return &nodeEvent{
Event: NodeEventType(ne.Event),
Node: DecodeNode(ne.Node),
}
}
// Nodes represents a list of nodes.
type Nodes []*Node
@ -265,7 +219,7 @@ type cluster struct {
func newCluster() *cluster {
return &cluster{
Hasher: &jmphasher{},
partitionN: DefaultPartitionN,
partitionN: defaultPartitionN,
ReplicaN: 1,
joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel
@ -273,7 +227,7 @@ func newCluster() *cluster {
closing: make(chan struct{}),
joining: make(chan struct{}),
InternalClient: NewNopInternalClient(),
InternalClient: newNopInternalClient(),
logger: NopLogger,
}
@ -313,8 +267,8 @@ func (c *cluster) setCoordinator(n *Node) error {
c.mu.Unlock()
// Send the update coordinator message to all nodes.
err := c.broadcaster.SendSync(
&internal.UpdateCoordinatorMessage{
New: EncodeNode(n),
&UpdateCoordinatorMessage{
New: n,
})
if err != nil {
return fmt.Errorf("problem sending UpdateCoordinator message: %v", err)
@ -369,7 +323,7 @@ func (c *cluster) addNode(node *Node) error {
if c.Topology == nil {
return fmt.Errorf("Cluster.Topology is nil")
}
if !c.Topology.AddID(node.ID) {
if !c.Topology.addID(node.ID) {
return nil
}
@ -389,7 +343,7 @@ func (c *cluster) removeNode(node *Node) error {
if c.Topology == nil {
return fmt.Errorf("Cluster.Topology is nil")
}
if !c.Topology.RemoveID(node.ID) {
if !c.Topology.removeID(node.ID) {
return nil
}
@ -410,7 +364,7 @@ func (c *cluster) setID(id string) {
c.id = id
// Make sure the Topology is updated.
c.Topology.ClusterID = c.id
c.Topology.clusterID = c.id
}
func (c *cluster) State() string {
@ -468,7 +422,7 @@ func (c *cluster) setNodeState(state string) error {
}
// Send node state to coordinator.
ns := &internal.NodeStateMessage{
ns := &NodeStateMessage{
NodeID: c.Node.ID,
State: state,
}
@ -505,12 +459,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
return nil
}
// Status returns the internal ClusterStatus representation.
func (c *cluster) Status() *internal.ClusterStatus {
return &internal.ClusterStatus{
// Status returns the the cluster's status including what nodes it contains, its ID, and current state.
func (c *cluster) Status() *ClusterStatus {
return &ClusterStatus{
ClusterID: c.id,
State: c.state,
Nodes: EncodeNodes(c.Nodes),
Nodes: c.Nodes,
}
}
@ -685,8 +639,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error)
// fragSources returns a list of ResizeSources - for each node in the `to` cluster -
// required to move from cluster `c` to cluster `to`.
func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) {
m := make(map[string][]*internal.ResizeSource)
func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) {
m := make(map[string][]*ResizeSource)
// Determine if a node is being added or removed.
action, diffNodeID, err := c.diff(to)
@ -745,7 +699,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
// Get the ResizeSource for each diff.
for nodeID, diff := range diffs {
m[nodeID] = []*internal.ResizeSource{}
m[nodeID] = []*ResizeSource{}
for _, frag := range diff {
// If there is no valid source node ID for a fragment,
// it likely means that the replica factor was not
@ -756,8 +710,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)")
}
src := &internal.ResizeSource{
Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)),
src := &ResizeSource{
Node: c.unprotectedNodeByID(srcNodeID),
Index: idx.Name(),
Field: frag.field,
View: frag.view,
@ -864,7 +818,7 @@ func (c *cluster) setup() error {
return errors.Wrap(err, "loading topology")
}
c.id = c.Topology.ClusterID
c.id = c.Topology.clusterID
// Only the coordinator needs to consider the .topology file.
if c.isCoordinator() {
@ -901,9 +855,9 @@ func (c *cluster) waitForStarted() error {
// TODO: Because the normal code path already sends a NodeJoin event (via
// memberlist), this it a bit redundant in most cases. Perhaps determine
// that the node has been restarted and don't do this step.
msg := &internal.NodeEventMessage{
Event: uint32(NodeJoin),
Node: EncodeNode(c.Node),
msg := &NodeEvent{
Event: NodeJoin,
Node: c.Node,
}
if err := c.broadcaster.SendSync(msg); err != nil {
return fmt.Errorf("sending restart NodeJoin: %v", err)
@ -934,22 +888,22 @@ func (c *cluster) markAsJoined() {
}
func (c *cluster) needTopologyAgreement() bool {
return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
}
func (c *cluster) haveTopologyAgreement() bool {
if c.Static {
return true
}
return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs())
return stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
}
func (c *cluster) allNodesReady() bool {
if c.Static {
return true
}
for _, uri := range c.Topology.NodeIDs {
if c.Topology.nodeStates[uri] != NodeStateReady {
for _, uri := range c.Topology.nodeIDs {
if c.Topology.nodeStates[uri] != nodeStateReady {
return false
}
}
@ -1013,8 +967,8 @@ func (c *cluster) setStateAndBroadcast(state string) error {
return c.broadcaster.SendSync(c.Status())
}
func (c *cluster) sendTo(node *Node, msg proto.Message) error {
if err := c.broadcaster.SendTo(node, msg); err != nil {
func (c *cluster) sendTo(node *Node, m Message) error {
if err := c.broadcaster.SendTo(node, m); err != nil {
return errors.Wrap(err, "sending")
}
return nil
@ -1120,7 +1074,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
}
// multiIndex is a map of sources initialized with all the nodes in toCluster.
multiIndex := make(map[string][]*internal.ResizeSource)
multiIndex := make(map[string][]*ResizeSource)
for _, n := range toCluster.Nodes {
multiIndex[n.ID] = nil
@ -1144,12 +1098,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
j.IDs[id] = true
continue
}
instr := &internal.ResizeInstruction{
instr := &ResizeInstruction{
JobID: j.ID,
Node: EncodeNode(toCluster.unprotectedNodeByID(id)),
Coordinator: EncodeNode(c.coordinatorNode()),
Node: toCluster.unprotectedNodeByID(id),
Coordinator: c.coordinatorNode(),
Sources: sources,
Schema: c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node.
Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node.
ClusterStatus: c.Status(),
}
j.Instructions = append(j.Instructions, instr)
@ -1175,7 +1129,7 @@ func (c *cluster) completeCurrentJob(state string) error {
}
// followResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error {
func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
c.logger.Printf("follow resize instruction on %s", c.Node.ID)
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
@ -1193,7 +1147,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
<-c.holder.opened
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
@ -1212,7 +1166,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
for _, src := range instr.Sources {
c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
srcURI := decodeURI(src.Node.URI)
srcURI := src.Node.URI
// Retrieve field.
f := c.holder.Field(src.Index, src.Field)
@ -1264,14 +1218,14 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
complete.Error = err.Error()
}
if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil {
if err := c.sendTo(instr.Coordinator, complete); err != nil {
c.logger.Printf("sending resizeInstructionComplete error: err=%s", err)
}
}()
return nil
}
func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error {
j := c.job(complete.JobID)
@ -1308,7 +1262,7 @@ func (c *cluster) job(id int64) *resizeJob {
type resizeJob struct {
ID int64
IDs map[string]bool
Instructions []*internal.ResizeInstruction
Instructions []*ResizeInstruction
Broadcaster broadcaster
action string
@ -1411,7 +1365,7 @@ func (j *resizeJob) distributeResizeInstructions() error {
// a dummy node object to use in the SendTo() method.
node := &Node{
ID: instr.Node.ID,
URI: decodeURI(instr.Node.URI),
URI: instr.Node.URI,
}
j.Logger.Printf("send resize instructions: %v", instr)
if err := j.Broadcaster.SendTo(node, instr); err != nil {
@ -1421,14 +1375,14 @@ func (j *resizeJob) distributeResizeInstructions() error {
return nil
}
type NodeIDs []string
type nodeIDs []string
func (n NodeIDs) Len() int { return len(n) }
func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] }
func (n nodeIDs) Len() int { return len(n) }
func (n nodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
func (n nodeIDs) Less(i, j int) bool { return n[i] < n[j] }
// ContainsID returns true if idi matches one of the nodesets's IDs.
func (n NodeIDs) ContainsID(id string) bool {
func (n nodeIDs) ContainsID(id string) bool {
for _, nid := range n {
if nid == id {
return true
@ -1440,16 +1394,16 @@ func (n NodeIDs) ContainsID(id string) bool {
// Topology represents the list of hosts in the cluster.
type Topology struct {
mu sync.RWMutex
NodeIDs []string
nodeIDs []string
ClusterID string
clusterID string
// nodeStates holds the state of each node according to
// the coordinator. Used during startup and data load.
nodeStates map[string]string
}
func NewTopology() *Topology {
func newTopology() *Topology {
return &Topology{
nodeStates: make(map[string]string),
}
@ -1463,11 +1417,11 @@ func (t *Topology) ContainsID(id string) bool {
}
func (t *Topology) containsID(id string) bool {
return NodeIDs(t.NodeIDs).ContainsID(id)
return nodeIDs(t.nodeIDs).ContainsID(id)
}
func (t *Topology) positionByID(nodeID string) int {
for i, tid := range t.NodeIDs {
for i, tid := range t.nodeIDs {
if tid == nodeID {
return i
}
@ -1475,25 +1429,25 @@ func (t *Topology) positionByID(nodeID string) int {
return -1
}
// AddID adds the node ID to the topology and returns true if added.
func (t *Topology) AddID(nodeID string) bool {
// addID adds the node ID to the topology and returns true if added.
func (t *Topology) addID(nodeID string) bool {
t.mu.Lock()
defer t.mu.Unlock()
if t.containsID(nodeID) {
return false
}
t.NodeIDs = append(t.NodeIDs, nodeID)
t.nodeIDs = append(t.nodeIDs, nodeID)
sort.Slice(t.NodeIDs,
sort.Slice(t.nodeIDs,
func(i, j int) bool {
return t.NodeIDs[i] < t.NodeIDs[j]
return t.nodeIDs[i] < t.nodeIDs[j]
})
return true
}
// RemoveID removes the node ID from the topology and returns true if removed.
func (t *Topology) RemoveID(nodeID string) bool {
// removeID removes the node ID from the topology and returns true if removed.
func (t *Topology) removeID(nodeID string) bool {
t.mu.Lock()
defer t.mu.Unlock()
@ -1502,15 +1456,15 @@ func (t *Topology) RemoveID(nodeID string) bool {
return false
}
copy(t.NodeIDs[i:], t.NodeIDs[i+1:])
t.NodeIDs[len(t.NodeIDs)-1] = ""
t.NodeIDs = t.NodeIDs[:len(t.NodeIDs)-1]
copy(t.nodeIDs[i:], t.nodeIDs[i+1:])
t.nodeIDs[len(t.nodeIDs)-1] = ""
t.nodeIDs = t.nodeIDs[:len(t.nodeIDs)-1]
return true
}
// Encode converts t into its internal representation.
func (t *Topology) Encode() *internal.Topology {
// encode converts t into its internal representation.
func (t *Topology) encode() *internal.Topology {
return encodeTopology(t)
}
@ -1518,7 +1472,7 @@ func (t *Topology) Encode() *internal.Topology {
func (c *cluster) loadTopology() error {
buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology"))
if os.IsNotExist(err) {
c.Topology = NewTopology()
c.Topology = newTopology()
return nil
} else if err != nil {
return errors.Wrap(err, "reading file")
@ -1552,38 +1506,12 @@ func (c *cluster) saveTopology() error {
return nil
}
func encodeTopology(topology *Topology) *internal.Topology {
if topology == nil {
return nil
}
return &internal.Topology{
ClusterID: topology.ClusterID,
NodeIDs: topology.NodeIDs,
}
}
func decodeTopology(topology *internal.Topology) (*Topology, error) {
if topology == nil {
return nil, nil
}
t := NewTopology()
t.ClusterID = topology.ClusterID
t.NodeIDs = topology.NodeIDs
sort.Slice(t.NodeIDs,
func(i, j int) bool {
return t.NodeIDs[i] < t.NodeIDs[j]
})
return t, nil
}
func (c *cluster) considerTopology() error {
// Create ClusterID if one does not already exist.
if c.id == "" {
u := uuid.NewV4()
c.id = u.String()
c.Topology.ClusterID = c.id
c.Topology.clusterID = c.id
}
if c.Static {
@ -1591,13 +1519,13 @@ func (c *cluster) considerTopology() error {
}
// If there is no .topology file, it's safe to proceed.
if len(c.Topology.NodeIDs) == 0 {
if len(c.Topology.nodeIDs) == 0 {
return nil
}
// The local node (coordinator) must be in the .topology.
if !c.Topology.ContainsID(c.Node.ID) {
return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.NodeIDs)
return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.nodeIDs)
}
// If local node is the only thing in .topology, continue.
@ -1611,7 +1539,7 @@ func (c *cluster) considerTopology() error {
}
// ReceiveEvent represents an implementation of EventHandler.
func (c *cluster) ReceiveEvent(e *nodeEvent) error {
func (c *cluster) ReceiveEvent(e *NodeEvent) error {
// Ignore events sent from this node.
if e.Node.ID == c.Node.ID {
return nil
@ -1751,7 +1679,7 @@ func (c *cluster) nodeLeave(node *Node) error {
return nil
}
func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.logger.Printf("merge cluster status: %v", cs)
@ -1763,7 +1691,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
// Set ClusterID.
c.setID(cs.ClusterID)
officialNodes := DecodeNodes(cs.Nodes)
officialNodes := cs.Nodes
// Add all nodes from the coordinator.
for _, node := range officialNodes {
@ -1812,3 +1740,120 @@ func (c *cluster) setStatic(hosts []string) error {
}
return nil
}
type ClusterStatus struct {
ClusterID string
State string
Nodes []*Node
}
type ResizeInstruction struct {
JobID int64
Node *Node
Coordinator *Node
Sources []*ResizeSource
Schema *Schema
ClusterStatus *ClusterStatus
}
type ResizeSource struct {
Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"`
View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"`
Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
}
// Schema contains information about indexes and their configuration.
type Schema struct {
Indexes []*IndexInfo
}
func encodeTopology(topology *Topology) *internal.Topology {
if topology == nil {
return nil
}
return &internal.Topology{
ClusterID: topology.clusterID,
NodeIDs: topology.nodeIDs,
}
}
func decodeTopology(topology *internal.Topology) (*Topology, error) {
if topology == nil {
return nil, nil
}
t := newTopology()
t.clusterID = topology.ClusterID
t.nodeIDs = topology.NodeIDs
sort.Slice(t.nodeIDs,
func(i, j int) bool {
return t.nodeIDs[i] < t.nodeIDs[j]
})
return t, nil
}
type CreateShardMessage struct {
Index string
Shard uint64
}
type CreateIndexMessage struct {
Index string
Meta *IndexOptions
}
type DeleteIndexMessage struct {
Index string
}
type CreateFieldMessage struct {
Index string
Field string
Meta *FieldOptions
}
type DeleteFieldMessage struct {
Index string
Field string
}
type CreateViewMessage struct {
Index string
Field string
View string
}
type DeleteViewMessage struct {
Index string
Field string
View string
}
type ResizeInstructionComplete struct {
JobID int64
Node *Node
Error string
}
type SetCoordinatorMessage struct {
New *Node
}
type UpdateCoordinatorMessage struct {
New *Node
}
type NodeStateMessage struct {
NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
}
type NodeStatus struct {
Node *Node
MaxShards map[string]uint64
Schema *Schema
}
type RecalculateCaches struct{}

73
cluster_internal_test.go
View file

@ -24,7 +24,6 @@ import (
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) {
from *cluster
to *cluster
idx *Index
expected map[string][]*internal.ResizeSource
expected map[string][]*ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{},
"node1": []*internal.ResizeSource{},
"node2": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{},
"node1": []*ResizeSource{},
"node2": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) {
from: c4,
to: c3,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{
{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
"node1": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) {
from: c5,
to: c4,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{
{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
"node1": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
},
"node2": []*internal.ResizeSource{},
"node2": []*ResizeSource{},
},
err: "",
},
@ -537,12 +536,12 @@ func TestCluster_ResizeStates(t *testing.T) {
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
nodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
}
// Close TestCluster.
@ -559,7 +558,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
nodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
@ -587,7 +586,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
nodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
@ -626,14 +625,14 @@ func TestCluster_ResizeStates(t *testing.T) {
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Close TestCluster.
@ -649,7 +648,7 @@ func TestCluster_ResizeStates(t *testing.T) {
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
nodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
@ -722,14 +721,14 @@ func TestCluster_ResizeStates(t *testing.T) {
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Bits

12
cmd/check.go
View file

@ -25,10 +25,10 @@ import (
"github.com/pilosa/pilosa/ctl"
)
var Checker *ctl.CheckCommand
var checker *ctl.CheckCommand
func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
func newCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr)
checkCmd := &cobra.Command{
Use: "check <path> [path2]...",
Short: "Do a consistency check on a pilosa data file.",
@ -39,8 +39,8 @@ Performs a consistency check on data files.
if len(args) == 0 {
return fmt.Errorf("path required")
}
Checker.Paths = args
if err := Checker.Run(context.Background()); err != nil {
checker.Paths = args
if err := checker.Run(context.Background()); err != nil {
return err
}
return nil
@ -50,5 +50,5 @@ Performs a consistency check on data files.
}
func init() {
subcommandFns["check"] = NewCheckCommand
subcommandFns["check"] = newCheckCommand
}

12
cmd/config.go
View file

@ -25,10 +25,10 @@ import (
"github.com/pilosa/pilosa/server"
)
var Conf *ctl.ConfigCommand
var conf *ctl.ConfigCommand
func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
Server := server.NewCommand(stdin, stdout, stderr)
confCmd := &cobra.Command{
Use: "config",
@ -36,8 +36,8 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
Long: `config prints the current configuration to stdout`,
RunE: func(cmd *cobra.Command, args []string) error {
Conf.Config = Server.Config
if err := Conf.Run(context.Background()); err != nil {
conf.Config = Server.Config
if err := conf.Run(context.Background()); err != nil {
return err
}
return nil
@ -51,5 +51,5 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
}
func init() {
subcommandFns["config"] = NewConfigCommand
subcommandFns["config"] = newConfigCommand
}

4
cmd/export.go
View file

@ -26,7 +26,7 @@ import (
var Exporter *ctl.ExportCommand
func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr)
exportCmd := &cobra.Command{
Use: "export",
@ -60,5 +60,5 @@ The file does not contain any headers.
}
func init() {
subcommandFns["export"] = NewExportCommand
subcommandFns["export"] = newExportCommand
}

10
cmd/generate_config.go
View file

@ -24,17 +24,17 @@ import (
"github.com/pilosa/pilosa/ctl"
)
var GenerateConf *ctl.GenerateConfigCommand
var generateConf *ctl.GenerateConfigCommand
func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
func newGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
confCmd := &cobra.Command{
Use: "generate-config",
Short: "Print the default configuration.",
Long: `generate-config prints the default configuration to stdout
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := GenerateConf.Run(context.Background()); err != nil {
if err := generateConf.Run(context.Background()); err != nil {
return err
}
return nil
@ -45,5 +45,5 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.
}
func init() {
subcommandFns["generate-config"] = NewGenerateConfigCommand
subcommandFns["generate-config"] = newGenerateConfigCommand
}

6
cmd/import.go
View file

@ -25,8 +25,8 @@ import (
var Importer *ctl.ImportCommand
// NewImportCommand runs the Pilosa import subcommand for ingesting bulk data.
func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
// newImportCommand runs the Pilosa import subcommand for ingesting bulk data.
func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Importer = ctl.NewImportCommand(stdin, stdout, stderr)
importCmd := &cobra.Command{
Use: "import",
@ -67,5 +67,5 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
}
func init() {
subcommandFns["import"] = NewImportCommand
subcommandFns["import"] = newImportCommand
}

12
cmd/inspect.go
View file

@ -25,10 +25,10 @@ import (
"github.com/pilosa/pilosa/ctl"
)
var Inspector *ctl.InspectCommand
var inspector *ctl.InspectCommand
func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr)
inspectCmd := &cobra.Command{
Use: "inspect",
@ -42,8 +42,8 @@ Inspects a data file and provides stats.
} else if len(args) > 1 {
return fmt.Errorf("only one path allowed")
}
Inspector.Path = args[0]
if err := Inspector.Run(context.Background()); err != nil {
inspector.Path = args[0]
if err := inspector.Run(context.Background()); err != nil {
return err
}
return nil
@ -53,5 +53,5 @@ Inspects a data file and provides stats.
}
func init() {
subcommandFns["inspect"] = NewInspectCommand
subcommandFns["inspect"] = newInspectCommand
}

6
cmd/server.go
View file

@ -27,8 +27,8 @@ import (
// Server is global so that tests can control and verify it.
var Server *server.Command
// NewServeCmd creates a pilosa server and runs it with command line flags.
func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
// newServeCmd creates a pilosa server and runs it with command line flags.
func newServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Server = server.NewCommand(stdin, stdout, stderr)
serveCmd := &cobra.Command{
Use: "server",
@ -52,5 +52,5 @@ on the configured port.`,
}
func init() {
subcommandFns["server"] = NewServeCmd
subcommandFns["server"] = newServeCmd
}

4
ctl/common.go
View file

@ -36,8 +36,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP
flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)")
}
// CommandClient returns a pilosa.InternalHTTPClient for the command
func CommandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) {
// commandClient returns a pilosa.InternalHTTPClient for the command
func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) {
tlsConfig := cmd.TLSConfiguration()
var TLSConfig *tls.Config
if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {

2
ctl/export.go
View file

@ -75,7 +75,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Create a client to the server.
client, err := CommandClient(cmd)
client, err := commandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}

20
ctl/import.go
View file

@ -41,7 +41,7 @@ type ImportCommand struct {
Field string `json:"field"`
// Options for index & field to be created if they don't exist
IndexOptions pilosa.IndexOptions
indexOptions pilosa.IndexOptions
// CreateSchema ensures the schema exists before import
CreateSchema bool
@ -59,7 +59,7 @@ type ImportCommand struct {
Sort bool `json:"sort"`
// Reusable client.
Client pilosa.InternalClient `json:"-"`
client pilosa.InternalClient `json:"-"`
// Standard input/output
*pilosa.CmdIO
@ -89,11 +89,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
return errors.New("path required")
}
// Create a client to the server.
client, err := CommandClient(cmd)
client, err := commandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
cmd.Client = client
cmd.client = client
if cmd.CreateSchema {
err := cmd.ensureSchema(ctx)
@ -104,7 +104,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Determine the field type in order to correctly handle the input data.
fieldType := pilosa.DefaultFieldType
schema, err := cmd.Client.Schema(ctx)
schema, err := cmd.client.Schema(ctx)
if err != nil {
return errors.Wrap(err, "getting schema")
}
@ -130,11 +130,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
}
func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions)
err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions)
if err != nil {
return fmt.Errorf("Error Creating Index: %s", err)
}
err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field)
err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field)
if err != nil {
return fmt.Errorf("Error Creating Field: %s", err)
}
@ -254,7 +254,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
}
logger.Printf("importing shard: %d, n=%d", shard, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil {
if err := cmd.client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}
@ -351,7 +351,7 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
// TODO: does it help to sort the rowKeys?
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil {
if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil {
return errors.Wrap(err, "importing keys")
}
@ -448,7 +448,7 @@ func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldV
}
logger.Printf("importing shard: %d, n=%d", shard, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil {
if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil {
return errors.Wrap(err, "importing values")
}
}

56
diagnostics.go
View file

@ -37,8 +37,8 @@ type versionResponse struct {
Message string `json:"message"`
}
// DiagnosticsCollector represents a collector/sender of diagnostics data.
type DiagnosticsCollector struct {
// diagnosticsCollector represents a collector/sender of diagnostics data.
type diagnosticsCollector struct {
mu sync.Mutex
host string
VersionURL string
@ -56,9 +56,9 @@ type DiagnosticsCollector struct {
server *Server
}
// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
return &DiagnosticsCollector{
// newDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port".
func newDiagnosticsCollector(host string) *diagnosticsCollector {
return &diagnosticsCollector{
host: host,
VersionURL: defaultVersionCheckURL,
startTime: time.Now().Unix(),
@ -70,13 +70,13 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector {
}
// SetVersion of locally running Pilosa Cluster to check against master.
func (d *DiagnosticsCollector) SetVersion(v string) {
func (d *diagnosticsCollector) SetVersion(v string) {
d.version = v
d.Set("Version", v)
}
// Flush sends the current metrics.
func (d *DiagnosticsCollector) Flush() error {
func (d *diagnosticsCollector) Flush() error {
d.mu.Lock()
defer d.mu.Unlock()
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
@ -99,7 +99,7 @@ func (d *DiagnosticsCollector) Flush() error {
}
// CheckVersion of the local build against Pilosa master.
func (d *DiagnosticsCollector) CheckVersion() error {
func (d *diagnosticsCollector) CheckVersion() error {
var rsp versionResponse
req, err := http.NewRequest("GET", d.VersionURL, nil)
if err != nil {
@ -131,7 +131,7 @@ func (d *DiagnosticsCollector) CheckVersion() error {
}
// compareVersion check version strings.
func (d *DiagnosticsCollector) compareVersion(value string) error {
func (d *diagnosticsCollector) compareVersion(value string) error {
currentVersion := versionSegments(value)
localVersion := versionSegments(d.version)
@ -147,12 +147,12 @@ func (d *DiagnosticsCollector) compareVersion(value string) error {
}
// Encode metrics maps into the json message format.
func (d *DiagnosticsCollector) encode() ([]byte, error) {
func (d *diagnosticsCollector) encode() ([]byte, error) {
return json.Marshal(d.metrics)
}
// Set adds a key value metric.
func (d *DiagnosticsCollector) Set(name string, value interface{}) {
func (d *diagnosticsCollector) Set(name string, value interface{}) {
switch v := value.(type) {
case string:
if v == "" {
@ -166,7 +166,7 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) {
}
// logErr logs the error and returns true if an error exists
func (d *DiagnosticsCollector) logErr(err error) bool {
func (d *diagnosticsCollector) logErr(err error) bool {
if err != nil {
d.Logger.Printf("%v", err)
return true
@ -175,7 +175,7 @@ func (d *DiagnosticsCollector) logErr(err error) bool {
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithOSInfo() {
func (d *diagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.systemInfo.Uptime()
if !d.logErr(err) {
d.Set("HostUptime", uptime)
@ -199,7 +199,7 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() {
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
func (d *diagnosticsCollector) EnrichWithMemoryInfo() {
memFree, err := d.server.systemInfo.MemFree()
if !d.logErr(err) {
d.Set("MemFree", memFree)
@ -215,7 +215,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
}
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
func (d *diagnosticsCollector) EnrichWithSchemaProperties() {
var numShards uint64
numFields := 0
numIndexes := 0
@ -267,51 +267,51 @@ type SystemInfo interface {
MemUsed() (uint64, error)
}
// NewNopSystemInfo creates a no-op implementation of SystemInfo.
func NewNopSystemInfo() *NopSystemInfo {
return &NopSystemInfo{}
// newNopSystemInfo creates a no-op implementation of SystemInfo.
func newNopSystemInfo() *nopSystemInfo {
return &nopSystemInfo{}
}
// NopSystemInfo is a no-op implementation of SystemInfo.
type NopSystemInfo struct {
// nopSystemInfo is a no-op implementation of SystemInfo.
type nopSystemInfo struct {
}
// Uptime is a no-op implementation of SystemInfo.Uptime.
func (n *NopSystemInfo) Uptime() (uint64, error) {
func (n *nopSystemInfo) Uptime() (uint64, error) {
return 0, nil
}
// Platform is a no-op implementation of SystemInfo.Platform.
func (n *NopSystemInfo) Platform() (string, error) {
func (n *nopSystemInfo) Platform() (string, error) {
return "", nil
}
// Family is a no-op implementation of SystemInfo.Family.
func (n *NopSystemInfo) Family() (string, error) {
func (n *nopSystemInfo) Family() (string, error) {
return "", nil
}
// OSVersion is a no-op implementation of SystemInfo.OSVersion.
func (n *NopSystemInfo) OSVersion() (string, error) {
func (n *nopSystemInfo) OSVersion() (string, error) {
return "", nil
}
// KernelVersion is a no-op implementation of SystemInfo.KernelVersion.
func (n *NopSystemInfo) KernelVersion() (string, error) {
func (n *nopSystemInfo) KernelVersion() (string, error) {
return "", nil
}
// MemFree is a no-op implementation of SystemInfo.MemFree.
func (n *NopSystemInfo) MemFree() (uint64, error) {
func (n *nopSystemInfo) MemFree() (uint64, error) {
return 0, nil
}
// MemTotal is a no-op implementation of SystemInfo.MemTotal.
func (n *NopSystemInfo) MemTotal() (uint64, error) {
func (n *nopSystemInfo) MemTotal() (uint64, error) {
return 0, nil
}
// MemUsed is a no-op implementation of SystemInfo.MemUsed.
func (n *NopSystemInfo) MemUsed() (uint64, error) {
func (n *nopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}

8
diagnostics_internal_test.go
View file

@ -29,7 +29,7 @@ func TestDiagnosticsClient(t *testing.T) {
server := httptest.NewServer(nil)
// Create a new client.
d := NewDiagnosticsCollector(server.URL)
d := newDiagnosticsCollector(server.URL)
d.Set("gg", 10)
d.Set("ss", "ss")
@ -76,7 +76,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) {
}
func TestDiagnosticsVersion_Compare(t *testing.T) {
d := NewDiagnosticsCollector("localhost:10101")
d := newDiagnosticsCollector("localhost:10101")
version := "v0.1.1"
d.SetVersion(version)
@ -118,7 +118,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
}))
// Create a new client.
d := NewDiagnosticsCollector("localhost:10101")
d := newDiagnosticsCollector("localhost:10101")
version := "0.1.1"
d.SetVersion(version)
@ -143,7 +143,7 @@ func BenchmarkDiagnostics(b *testing.B) {
server := httptest.NewServer(nil)
// Create a new client.
d := NewDiagnosticsCollector(server.URL)
d := newDiagnosticsCollector(server.URL)
prev := runtime.GOMAXPROCS(4)
defer runtime.GOMAXPROCS(prev)

56
docs/administration.md
View file

@ -24,19 +24,19 @@ Pilosa holds all row/column bitmap data in main memory. While this data is compr
#### CPUs
Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [slice](../data-model/#slice), so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload.
Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload.
#### Disk
Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs—especially if you have a write heavy application.
Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application.
#### Network
Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there already should be a system of record, or ability to rebuild a cluster quickly from backups.
Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups.
#### Overview
While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
### Open File Limits
@ -56,23 +56,23 @@ When importing large datasets remember it is much faster to pre sort the data by
pilosa import --sort -i project -f stargazer project-stargazer.csv
```
##### Importing Field Values
##### Importing Integer Values
If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `Column,Value`.
If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`.
```
pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv
pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
```
<div class="note">
<p>Note that you must first create a frame and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
<p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
</div>
#### Exporting
Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column.
Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column.
```request
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \
curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \
--header "Accept: text/csv"
```
```response
@ -122,7 +122,7 @@ Pilosa v0.9 introduces a few compatibility changes that need to be addressed.
Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called [`topology-generator`](https://github.com/pilosa/upgrade-utils/tree/master/v0.9/topology-generator):
1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical.
2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`.
3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`.
4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position.
@ -132,8 +132,8 @@ Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topo
**Application changes**:
1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported.
2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed.
3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release.
2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-field, as index-level time-quantums have been removed.
3. Inverse fields have been deprecated, removed from docs, and will be unsupported in the next release.
### Resizing the Cluster
@ -211,7 +211,7 @@ curl localhost:10101/cluster/resize/set-coordinator \
### Backup/restore
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster.
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster.
Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node.
@ -230,12 +230,12 @@ Note: This will only work when the replication factor is >= 2
- To accomplish this you will first need:
- List of all indexes on your cluster
- List of all frames in your indexes
- Max slice per index, listed in the `/slices/max` endpoint
- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each slice
- Using the list of slices owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Frame
- copy each owned slice for an existing node to this new node
- List of all fields in your indexes
- Max shard per index, listed in the `/internal/shards/max` endpoint
- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard
- Using the list of shards owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Field
- copy each owned shard for an existing node to this new node
- Modify the cluster config file to replace the previous node address with the new node address.
- Restart the cluster
- Wait for the first sync (10 minutes) to validate Index connections
@ -249,11 +249,11 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi
- **Cluster:** List of nodes in the cluster.
- **NumNodes:** Number of nodes in the cluster.
- **NumCPU:** Number of cores per node
- **BSIEnabled:** Bit Slice Index Frames in use.
- **TimeQuantumEnabled:** Time Quantum Frames in use.
- **BSIEnabled:** Bit Sliced Index Fields in use.
- **TimeQuantumEnabled:** Time Quantum Fields in use.
- **NumIndexes:** Number of indexes in the Cluster.
- **NumFrames:** Number of frames in the Cluster.
- **NumSlices:** Number of slices in the Cluster.
- **NumFields:** Number of fields in the Cluster.
- **NumShards:** Number of shards in the Cluster.
- **NumViews:** Number of views in the Cluster.
- **OpenFiles:** Open file handle count.
- **GoRoutines:** Go routine count.
@ -274,16 +274,16 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- NodeID
- Index
- Frame
- Field
- View
- Slice
- Shard
#### Events
We currently track the following events
- **Index:** The creation of a new index.
- **Frame:** The creation of a new frame.
- **MaxSlice:** The creation of a new Slice.
- **Field:** The creation of a new field.
- **MaxShard:** The creation of a new Shard.
- **SetBit:** Count of set bits.
- **ClearBit:** Count of cleared bits.
- **ImportBit:** During a bulk data import this represents the count of bits created.

75
docs/api-reference.md
View file

@ -17,7 +17,7 @@ Returns the schema of all indexes in JSON.
curl -XGET localhost:10101/index
```
``` response
{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
{"indexes":[{"name":"user","fields":[{"name":"collab"}]}]}
```
### List index schema
@ -30,7 +30,7 @@ Returns the schema of the specified index in JSON.
curl -XGET localhost:10101/index/user
```
``` response
{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
{"name":"user", "fields":[{"name":"collab"}]}
```
### Create index
@ -43,7 +43,7 @@ Creates an index with the given name.
curl -XPOST localhost:10101/index/user
```
``` response
{}
{"success":true}
```
### Remove index
@ -56,7 +56,7 @@ Removes the given index.
curl -XDELETE localhost:10101/index/user
```
``` response
{}
{"success":true}
```
### Query index
@ -68,102 +68,81 @@ Sends a [query](../query-language/) to the Pilosa server with the given index. T
``` request
curl localhost:10101/index/user/query \
-X POST \
-d 'Bitmap(frame="language", row=5)'
-d 'Row(language=5)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
{"results":[{"attrs":{},"columns":[100]}]}
```
In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices.
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \
-X POST \
-d 'Bitmap(frame="language", row=5)'
-d 'Row(language=5)'
```
``` response
{
"results":[{"attrs":{},"bits":[100]}],
"results":[{"attrs":{},"columns":[100]}],
"columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}]
}
```
By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
### Create frame
### Create field
`POST /index/<index-name>/frame/<frame-name>`
`POST /index/<index-name>/field/<field-name>`
Creates a frame in the given index with the given name.
Creates a field in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:
* `name` (string): Field name.
* `type` (string): Field type, currently only "int" is supported.
* `type` (string): Field type, "set", "int" or "time".
* `min` (int): Minimum value allowed for this field.
* `max` (int): Maximum value allowed for this field.
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
``` request
curl localhost:10101/index/user/frame/language -X POST
curl localhost:10101/index/user/field/language -X POST
```
``` response
{}
{"success":true}
```
``` request
curl localhost:10101/index/repository/frame/stats \
curl localhost:10101/index/repository/field/stats \
-X POST \
-d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
```
``` response
{}
{"success":true}
```
### Remove frame
### Remove field
`DELETE /index/<index-name>/frame/<frame-name>`
`DELETE /index/<index-name>/field/<field-name>`
Removes the given frame.
Removes the given field.
``` request
curl -XDELETE localhost:10101/index/user/frame/language
curl -XDELETE localhost:10101/index/user/field/language
```
``` response
{}
```
### Create Field
`POST /index/<index-name>/frame/<frame-name>/field/<field-name>`
Creates a new field to store integer values in the given frame.
The request payload is JSON, and it must contain the fields `type`, `min`, `max`.
* `type` (string): Field type, currently only "int" is supported.
* `min` (int): Minimum value allowed for this field.
* `max` (int): Maximum value allowed for this field.
``` request
curl localhost:10101/index/repository/frame/stats/field/pullrequests \
-X POST \
-d '{"type": "int", "min": 0, "max": 1000000}'
```
``` response
{}
{"success":true}
```
### Get version
@ -191,7 +170,7 @@ in a multi-node cluster, the cache is only recalculated on the node
that receives the request.
``` request
curl -XGET localhost:10101/recalculate-caches
curl -XPOST localhost:10101/recalculate-caches
```
Response: `204 No Content`

6
docs/client-libraries.md
View file

@ -90,7 +90,7 @@ func main() {
fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns)
// Set user 99999 as a stargazer for repository 77777?
client.Query(stargazer.SetBit(99999, 77777))
client.Query(stargazer.Set(99999, 77777))
}
```
@ -174,7 +174,7 @@ mutually_starred = client.query(query).result.row.columns
print("User 14 or 19 starred, written in language 1:", mutually_starred)
# Set user 99999 as a stargazer for repository 77777
client.query(stargazer.setbit(99999, 77777))
client.query(stargazer.set(99999, 77777))
```
Running the above program should produce output like this:
@ -275,7 +275,7 @@ public class StarTrace {
System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs);
// Set user 99999 as a stargazer for repository 77777:
client.query(stargazer.setBit(99999, 77777));
client.query(stargazer.set(99999, 77777));
}
}
```

2
docs/configuration.md
View file

@ -106,7 +106,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
#### Max Writes Per Request
* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue.
* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs.
* Flag: `--max-writes-per-request=5000`
* Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000`
* Config:

84
docs/data-model.md
View file

@ -6,10 +6,10 @@ nav = [
"Index",
"Column",
"Row",
"Frame",
"Field",
"Time Quantum",
"Attribute",
"Slice",
"Shard",
"View",
]
+++
@ -22,7 +22,7 @@ The central component of Pilosa's data model is a boolean matrix. Each cell in t
Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *frames* and quickly retrieves the top rows in a frame sorted by the number of bits set in each row.
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 2<sup>63</sup> on a single-node cluster, for example, will not work well due to memory limitations.
@ -35,20 +35,22 @@ The purpose of the Index is to represent a data namespace. You cannot perform cr
### Column
Column ids are sequential increasing integers and are common to all Frames within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
Column ids are sequential increasing integers and are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
### Row
Row ids are sequential increasing integers namespaced to each Frame within an Index.
Row ids are sequential increasing integers namespaced to each Field within an Index.
### Frame
### Field
Frames are used to segment rows within an index, for example to define different functional groups. A frame might correspond to a single field in a relational table, where each row in a standard frame represents a single possible value of the field. Similarly, a frame with BSI values could represent all possible integer values of a field .
Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field.
#### Relational Analogy
The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here.
TODO diagram showing a few rows of a relational table and corresponding pilosa index
Entities:
Relational | Pilosa
@ -56,19 +58,19 @@ Entities:
Database | N/A *(internal: Holder)*
Table | Index
Row | Column
Column | Frame
Column | Field
Value | Row
Value (int) | Field.Value (see [BSI](#bsi-range-encoding))
Simple queries:
Relational | Pilosa
---------------------------------------------|------------------------------------
`select ID from People where Name = 'Bob'` | `Bitmap(frame=Name, row=[Bob])`
`select ID from People where Age > 30` | `Range(frame=Default, Age > 30)`
`select ID from People where Member = true` | `Bitmap(frame=Member, row=[true])`
Relational | Pilosa
-----------------------------------------------|------------------------------------
`select ID from People where Name = 'Bob'` | `Row(Name="Bob")`
`select ID from People where Age > 30` | `Range(Age > 30)`
`select ID from People where Member = true` | `Row(Member=0)`
In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple frames. For example, this SQL join:
Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join:
```sql
select AVG(p.Age) from People p
@ -80,44 +82,44 @@ where c.Make = 'Ford'
can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed):
```pql
Sum(Bitmap(frame="Car-Make", row=[Ford]), frame=Default, field=Age)
Sum(Row(Car-Make="Ford"), field=Age)
```
This is one major component of Pilosa's ability to combine relationships from multiple data stores.
#### Ranked
Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation.
![ranked frame diagram](/img/docs/frame-ranked.svg)
*Ranked frame diagram*
![ranked field diagram](/img/docs/field-ranked.svg)
*Ranked field diagram*
#### LRU
The LRU cache maintains the most recently accessed Rows.
![lru frame diagram](/img/docs/frame-lru.svg)
*LRU frame diagram*
![lru field diagram](/img/docs/field-lru.svg)
*LRU field diagram*
### Time Quantum
Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
### Attribute
Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure.
Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all frames in an index. Row attributes apply to all bits in the corresponding row.
Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row.
### Slice
### Shard
Indexes are sharded into groups of columns called Slices. Each Slice contains a fixed number of columns, which is the SliceWidth. SliceWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
### View
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API.
Views represent the various data layouts within a Field. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API.
#### Standard
@ -125,34 +127,34 @@ The standard View contains the same Row/Column format as the input data.
#### Time Quantums
If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below:
If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
```
SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00")
Set(3, A=8, 2017-05-18T00:00)
Set(3, A=8, 2017-05-19T00:00)
```
![time quantum frame diagram](/img/docs/frame-time-quantum.svg)
*Time quantum frame diagram*
![time quantum field diagram](/img/docs/field-time-quantum.svg)
*Time quantum fueld diagram*
#### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
For example, the following `SetFieldValue()` queries will result in the data described in the diagram below:
For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below:
```
SetFieldValue(col=1, frame="A", field0=1)
SetFieldValue(col=2, frame="A", field0=2)
SetFieldValue(col=3, frame="A", field0=3)
SetFieldValue(col=4, frame="A", field0=7)
SetFieldValue(col=2, frame="A", field1=1)
SetFieldValue(col=3, frame="A", field1=6)
Set(1, A=1)
Set(2, A=2)
Set(3, A=3)
Set(4, A=7)
Set(2, B=1)
Set(3, B=6)
```
![BSI frame diagram](/img/docs/frame-bsi.svg)
*BSI frame diagram*
![BSI field diagram](/img/docs/field-bsi.svg)
*BSI field diagram*
Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.

46
docs/examples.md
View file

@ -3,7 +3,6 @@ title = "Examples"
weight = 4
nav = [
"Transportation",
"Chemical similarity search",
]
+++
@ -33,9 +32,9 @@ The NYC taxi data is comprised of a number of csv files listed here: http://www.
* Dropoff time: timestamp
* Pickup time: timestamp
We import these fields, creating one or more Pilosa frames from each of them:
We import these fields, creating one or more Pilosa fields from each of them:
frame |mapping
field |mapping
------------|---------------------
cab_type |direct map of enum int → row ID
dist_miles |round(dist) → row ID
@ -52,24 +51,24 @@ pickup_month |month(timestamp) → row ID
pickup_day |day(timestamp) → row ID
pickup_time |time of day mapped to one of 48 half-hour buckets → row ID
We also created two extra frames that represent the duration and average speed of each ride:
We also created two extra fields that represent the duration and average speed of each ride:
frame |mapping
field |mapping
--------------------|-------------
duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID
speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
#### Mapping
Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
##### 0 columns → 1 frame
##### 0 columns → 1 field
**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant.
**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant.
##### 1 column → 1 frame
##### 1 column → 1 field
The following three frames are mapped in a simple direct way from single columns of the original data.
The following three fields are mapped in a simple direct way from single columns of the original data.
**dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
@ -84,7 +83,7 @@ lfm := pdk.LinearFloatMapper{
`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`).
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`). TODO update so this makes sense
```go
pdk.BitMapper{
Frame: "dist_miles",
@ -129,27 +128,27 @@ Here, we define a list of Mappers, each including a name, which we use to refer
**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
##### 1 column → multiple frames
##### 1 column → multiple fields
When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day".
We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day".
We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day".
We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day".
We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
##### Multiple columns → 1 frame
##### Multiple columns → 1 field
The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id.
We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id.
Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
##### Complex mappings
We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
```go
durm := pdk.CustomMapper{
Func: func(fields ...interface{}) interface{} {
@ -172,7 +171,7 @@ Now we can run some example queries.
Count per cab type can be retrieved, sorted, with a single PQL call.
```request
TopN(frame=cab_type)
TopN(cab_type)
```
```response
{"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]}
@ -181,7 +180,7 @@ TopN(frame=cab_type)
High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
```request
TopN(frame=pickup_grid_id)
TopN(pickup_grid_id)
```
```response
{"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]}
@ -193,7 +192,7 @@ Average of `total_amount` per `passenger_count` can be computed with some postpr
queries = ''
pcounts = range(10)
for i in pcounts:
queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i
queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i
resp = requests.post(qurl, data=queries)
average_amounts = []
@ -209,6 +208,8 @@ Note that the <a href="../data-model/#bsi-range-encoding">BSI</a>-powered <a hre
For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
<!--
### Chemical similarity search
<div class="warning">
@ -326,3 +327,6 @@ python benchmarks.py -id 6223
As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster
-->

30
docs/getting-started.md
View file

@ -47,7 +47,7 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term
#### Create the Schema
Note:
The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
If at any time you want to verify the data structure, you can request the schema as follows:
``` request
curl localhost:10101/schema
@ -61,7 +61,7 @@ Before we can import data or run queries, we need to create our indexes and the
curl localhost:10101/index/repository -X POST
```
``` response
{}
{"success":true}
```
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
@ -71,10 +71,10 @@ curl localhost:10101/index/repository/field/stargazer \
-d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
```
``` response
{}
{"success":true}
```
Since our data contains time stamps for the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
Next up is the `language` field, which will contain IDs for programming languages:
``` request
@ -82,7 +82,7 @@ curl localhost:10101/index/repository/field/language \
-X POST
```
``` response
{}
{"success":true}
```
The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options.
@ -119,7 +119,7 @@ Which repositories did user 14 star:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Bitmap(field="stargazer", row=14)'
-d 'Row(stargazer=14)'
```
``` response
{
@ -136,7 +136,7 @@ What are the top 5 languages in the sample data:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'TopN(field="language", n=5)'
-d 'TopN(language, n=5)'
```
``` response
{
@ -157,8 +157,8 @@ Which repositories were starred by user 14 and 19:
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Bitmap(field="stargazer", row=14),
Bitmap(field="stargazer", row=19)
Row(stargazer=14),
Row(stargazer=19)
)'
```
``` response
@ -177,8 +177,8 @@ Which repositories were starred by user 14 or 19:
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(
Bitmap(field="stargazer", row=14),
Bitmap(field="stargazer", row=19)
Row(stargazer=14),
Row(stargazer=19)
)'
```
``` response
@ -197,9 +197,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Bitmap(field="stargazer", row=14),
Bitmap(field="stargazer", row=19),
Bitmap(field="language", row=1)
Row(stargazer=14),
Row(stargazer=19),
Row(language=1)
)'
```
``` response
@ -217,7 +217,7 @@ Set user 99999 as a stargazer for repository 77777:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(field="stargazer", col=77777, row=99999)'
-d 'Set(77777, stargazer=99999)'
```
``` response
{"results":[true]}

38
docs/glossary.md
View file

@ -6,25 +6,25 @@ nav = []
## Glossary
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column).
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column).
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap).
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index).
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index).
<strong id="field">[Field](../data-model/#bsi-range-encoding):</strong> A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of three types: set, [int](#bsi), and time. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.
<strong id="gossip">[Gossip](https://en.wikipedia.org/wiki/Gossip_protocol):</strong> A protocol used by Pilosa for internal communication.
@ -32,11 +32,11 @@ nav = []
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field).
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
<strong id="maxshard">MaxShard:</strong> The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5.
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field).
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field).
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
@ -54,20 +54,20 @@ nav = []
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="slice">[Slice](../data-model/#slice):</strong> [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
<strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
<strong id="slicewidth">SliceWidth:</strong> This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
<strong id="shard">[Shard](../data-model/#shard):</strong> [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field).
<strong id="shardwidth">ShardWidth:</strong> This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
<strong id="tanimoto">[Tanimoto](../examples/#chemical-similarity-search):</strong> Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|).
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for time [Range](#range) queries.
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [Range](#range) queries on time [fields](#field).
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field).
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

36
docs/pdk.md
View file

@ -18,7 +18,7 @@ Running `pdk -h` will give the most up to date list of all the tools and example
`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
record from Kafka is assigned a Pilosa column, and each value in a record is
assigned a row or field. Frame and field names are built from the "path" through
assigned a row or field. Pilosa field names are built from the "path" through
the record to arrive at that field. For example:
```json
@ -38,30 +38,30 @@ the record to arrive at that field. For example:
This JSON object would result in the following Pilosa schema:
| Name | Field | Type | Min | Max | Size |
|----------------|-----------|--------|-----|------------|--------|
| name | | ranked | | | 100000 |
| favorite_foods | | ranked | | | 100000 |
| default | | ranked | | | 100000 |
| | age | int | 0 | 2147483647 | |
| location | | ranked | | | 1000 |
| | latitude | int | 0 | 2147483647 | |
| | longitude | int | 0 | 2147483647 | |
| location-city | | ranked | | | 100000 |
| location-state | | ranked | | | 100000 |
| Field | Type | Min | Max | Size |
|----------------|--------|-----|------------|--------|
| name | ranked | | | 100000 |
| favorite_foods | ranked | | | 100000 |
| default | ranked | | | 100000 |
| age | int | 0 | 2147483647 | |
| location | ranked | | | 1000 |
| latitude | int | 0 | 2147483647 | |
| longitude | int | 0 | 2147483647 | |
| location-city | ranked | | | 100000 |
| location-state | ranked | | | 100000 |
All frames are created as ranked frames by default, with the cache size listed above. Fields are created with
a minimum size of zero and a fixed maximum of 2147483647. Fields at the top level
are created in the default frame. Frames are a dash-separated concatenation of
all key values in the path - you can see this with frames like location-city.
All set fields are created as ranked fields by default, with the cache size
listed above. Integer fields are created with a minimum size of zero and a
fixed maximum of 2147483647. Field names are a dash-separated concatenation of
all key values in the path - you can see this with fields like location-city.
Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
kafka topics, kafka group, etc.), but there are a few options that give some
control over the way data is indexed, and ingestion performance.
* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state".
* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state".
* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.

285
docs/query-language.md
View file

@ -1,4 +1,4 @@
+++
v+++
title = "Query Language"
weight = 6
nav = [
@ -29,13 +29,13 @@ There will be one item in the `results` array for each PQL query in the request.
##### Examples
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index and fields, and to populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would:
The examples just show the PQL quer(ies) needed - to run the query `Set(10, stargazer=1)` against a server using curl, you would:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", col=10, row=1)'
-d 'Set(10, stargazer=1)'
```
``` response
{"results":[true]}
@ -43,28 +43,27 @@ curl localhost:10101/index/repository/query \
#### Arguments and Types
* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04)
* `UINT` An unsigned integer (e.g. 42839)
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
* `ATTR_VALUE` Can be a string, float, integer, or bool.
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`
* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
### Write Operations
#### SetBit
#### Set
**Spec:**
```
SetBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
Set(<COLUMN>, <FIELD>=<ROW>, [TIMESTAMP])
```
**Description:**
`SetBit` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column.
`Set` assigns a value of 1 to a bit in the binary matrix, thus associating the given row (the `<ROW>` value) in the given field with the given column.
**Result Type:** boolean
@ -77,17 +76,17 @@ A return value of `false` indicates that the bit was already set to 1 and nothin
Set the bit at row 1, column 10:
```request
SetBit(frame="stargazer", col=10, row=1)
Set(10, stargazer=1)
```
```response
{"results":[true]}
```
This sets a bit in the stargazer frame, representing that the user with id=1 has starred the repository with id=10.
This sets a bit in the stargazer field, representing that the user with id=1 has starred the repository with id=10.
SetBit also supports providing a timestamp. To write the date that a user starred a repository:
Set also supports providing a timestamp. To write the date that a user starred a repository:
```request
SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
Set(10, stargazer=1, 2016-01-01T00:00)
```
```response
{"results":[true]}
@ -95,24 +94,32 @@ SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00")
Set multiple bits in a single request:
```request
SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2)
Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30)
```
```response
{"results":[false,true,true,true]}
```
Set the field "pullrequests" to integer value 2 at column 10:
```request
Set(10, pullrequests=2)
```
```response
{"results":[true]}
```
#### SetRowAttrs
**Spec:**
```
SetRowAttrs(<frame=STRING>, <row=UINT>,
SetRowAttrs(<FIELD>, <ROW>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
**Description:**
`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute.
`SetRowAttrs` associates arbitrary key/value pairs with a row in a field. Setting a value of `null`, without quotes, deletes an attribute.
**Result Type:** null
@ -122,17 +129,17 @@ SetRowAttrs queries always return `null` upon success.
Set attributes `username` and `active` on row 10:
```request
SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true)
SetRowAttrs(stargazer, 10, username="mrpi", active=true)
```
```response
{"results":[null]}
```
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", row=10)`.
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Row](../query-language/#row) query like so `Row(stargazer=10)`.
Delete attribute `username` on row 10:
```request
SetRowAttrs(frame="stargazer", row=10, username=null)
SetRowAttrs(stargazer, 10, username=null)
```
```response
{"results":[null]}
@ -143,7 +150,7 @@ SetRowAttrs(frame="stargazer", row=10, username=null)
**Spec:**
```
SetColumnAttrs(<frame=STRING>, <row=UINT>,
SetColumnAttrs(<COLUMN>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
@ -154,13 +161,13 @@ SetColumnAttrs(<frame=STRING>, <row=UINT>,
**Result Type:** null
SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. To avoid confusion, `frame` cannot be used as an attribute name.
SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute.
**Examples:**
Set attributes `stars`, `url`, and `active` on column 10:
```request
SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true)
SetColumnAttrs(10, stars=123, url="http://projects.pilosa.com/10", active=true)
```
```response
{"results":[null]}
@ -170,13 +177,13 @@ Set url value and active status for project 10. These are arbitrary key/value pa
ColumnAttrs can be requested by adding the URL parameter `columnAttrs=true` to a query. For example:
```request
curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Bitmap(frame="stargazer", row=1)Bitmap(frame="stargazer", row=2)'
curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Row(stargazer=1) Row(stargazer=2)'
```
```response
{
"results":[
{"attrs":{},"bits":[10,20]},
{"attrs":{},"bits":[10,30]}
{"attrs":{},"cols":[10,20]},
{"attrs":{},"cols":[10,30]}
],
"columnAttrs":[
{"id":10,"attrs":{"active":true,"stars":123,"url":"http://projects.pilosa.com/10"}},
@ -189,25 +196,25 @@ In this example, ColumnAttrs have been set on columns 10 and 20, but not column
Delete the `url` attribute on column 10:
```request
SetColumnAttrs(col=10, url=null)
SetColumnAttrs(10, url=null)
```
```response
{"results":[null]}
```
#### ClearBit
#### Clear
**Spec:**
```
ClearBit(<frame=STRING>, <row=UINT>, <col=UINT>)
Clear(<COLUMN>, <FIELD>=<ROW>)
```
**Description:**
`ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column.
`Clear` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column.
Note that clearing bits from time views is not supported.
Note that clearing a column on a time field will remove all data for that column.
**Result Type:** boolean
@ -217,9 +224,9 @@ A return value of `false` indicates that the bit was already set to 0 and nothin
**Examples:**
Clear the bit at row 1 and column 10 in the stargazer frame:
Clear the bit at row 1 and column 10 in the stargazer field:
```request
ClearBit(frame="stargazer", col=10, row=1)
Clear(10, stargazer=1)
```
```response
{"results":[true]}
@ -227,79 +234,48 @@ ClearBit(frame="stargazer", col=10, row=1)
This represents removing the relationship between the user with id=1 and the repository with id=10.
#### SetFieldValue
**Spec:**
```
SetFieldValue(<col=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
```
**Description:**
`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`.
**Result Type:** null
SetFieldValue returns `null` upon success.
**Examples:**
Set the field value `pullrequest` to the value 2, on column 10 in frame `stats`:
```request
SetFieldValue(col=10, frame="stats", pullrequests=2)
```
```response
{"results":[null]}
```
This represents setting the number of pull requests of repository 10 to 2.
This example assumes the existence of the frame `stats` and the field `pullrequests`. See [frame creation](../api-reference/#create-frame) and [field creation](../api-reference/#create-field) for more information.
### Read Operations
#### Bitmap
#### Row
**Spec:**
```
Bitmap(<frame=STRING>, (<rowL=UINT> | <col>=UINT))
Row(<FIELD>=<ROW>)
```
**Description:**
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column.
`Row` retrieves the indices of all the columns in a row. It also retrieves any attributes set on that row.
**Result Type:** object with attrs and bits.
**Result Type:** object with attrs and columns.
e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
e.g. `{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}`
**Examples:**
Query all columns with a bit set in row 1 of the frame `stargazer` (repositories that are starred by user 1):
Query all columns with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1):
```request
Bitmap(frame="stargazer", row=1)
Row(stargazer=1)
```
```response
{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}
{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}
```
* attrs are the attributes for user 1
* bits are the repositories which user 1 has starred.
* columns are the repositories which user 1 has starred.
#### Union
**Spec:**
```
Union([BITMAP_CALL ...])
Union([ROW_CALL ...])
```
**Description:**
Union performs a logical OR on the results of all `BITMAP_CALL` queries passed to it.
Union performs a logical OR on the results of all `ROW_CALL` queries passed to it.
**Result Type:** object with attrs and bits
@ -309,28 +285,27 @@ attrs will always be empty
Query columns with a bit set in either of two rows (repositories that are starred by either of two users):
```request
Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2))
Union(Row(stargazer=1), Row(stargazer=2))
```
```response
{"attrs":{},"bits":[10, 20, 30]}
{"attrs":{},"columns":[10, 20, 30]}
```
* bits are repositories that were starred by user 1 OR user 2
* columns are repositories that were starred by user 1 OR user 2
#### Intersect
**Spec:**
```
Intersect(<BITMAP_CALL>, [BITMAP_CALL ...])
Intersect(<ROW_CALL>, [ROW_CALL ...])
```
**Description:**
Intersect performs a logical AND on the results of all `BITMAP_CALL` queries passed to it.
Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it.
**Result Type:** object with attrs and bits
**Result Type:** object with attrs and columns
attrs will always be empty
@ -339,27 +314,27 @@ attrs will always be empty
Query columns with a bit set in both of two rows (repositories that are starred by both of two users):
```request
Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
Intersect(Row(stargazer=1), Row(stargazer=2))
```
```response
{"attrs":{},"bits":[10]}
{"attrs":{},"columns":[10]}
```
* bits are repositories that were starred by user 1 AND user 2
* columns are repositories that were starred by user 1 AND user 2
#### Difference
**Spec:**
```
Difference(<BITMAP_CALL>, [BITMAP_CALL ...])
Difference(<ROW_CALL>, [ROW_CALL ...])
```
**Description:**
Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`.
Difference returns all of the bits from the first `ROW_CALL` argument passed to it, without the bits from each subsequent `ROW_CALL`.
**Result Type:** object with attrs and bits
**Result Type:** object with attrs and columns
attrs will always be empty
@ -367,37 +342,37 @@ attrs will always be empty
Query columns with a bit set in one row and not another (repositories that are starred by one user and not another):
```request
Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2))
Difference(Row(stargazer=1), Row(stargazer=2))
```
```response
{"results":[{"attrs":{},"bits":[20]}]}
{"results":[{"attrs":{},"columns":[20]}]}
```
* bits are repositories that were starred by user 1 BUT NOT user 2
* columns are repositories that were starred by user 1 BUT NOT user 2
Query for the opposite difference:
```request
Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1))
Difference(Row(stargazer=2), Row(stargazer=1))
```
```response
{"attrs":{},"bits":[30]}
{"attrs":{},"columns":[30]}
```
* Bits are repositories that were starred by user 2 BUT NOT user 1
* columnss are repositories that were starred by user 2 BUT NOT user 1
#### Xor
**Spec:**
```
Xor(<BITMAP_CALL>, [BITMAP_CALL ...])
Xor(<ROW_CALL>, [ROW_CALL ...])
```
**Description:**
Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it.
Xor performs a logical XOR on the results of each `ROW_CALL` query passed to it.
**Result Type:** object with attrs and bits
**Result Type:** object with attrs and columns
attrs will always be empty
@ -406,24 +381,24 @@ attrs will always be empty
Query columns with a bit set in exactly one of two rows (repositories that are starred by only one of two users):
```request
Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2))
Xor(Row(stargazer=2), Row(stargazer=1))
```
```response
{"results":[{"attrs":{},"bits":[10,20,30]}]}
{"results":[{"attrs":{},"columns":[10,20,30]}]}
```
* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
* columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both)
#### Count
**Spec:**
```
Count(<BITMAP_CALL>)
Count(<ROW_CALL>)
```
**Description:**
Returns the number of set bits in the `BITMAP_CALL` passed in.
Returns the number of set bits in the `ROW_CALL` passed in.
**Result Type:** int
@ -431,7 +406,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in.
Query the number of bits set in a row (the number of repositories a user has starred):
```request
Count(Bitmap(frame="stargazer", row=1))
Count(Row(stargazer=1))
```
```response
{"results":[1]}
@ -444,34 +419,34 @@ Count(Bitmap(frame="stargazer", row=1))
**Spec:**
```
TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
[<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
TopN([ROW_CALL], <FIELD>, [n=UINT],
[attrName=<ATTR_NAME>, attrValues=<[]ATTR_VALUE>])
```
**Description:**
Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
The `field` and `filters` arguments work together to only return Bitmaps which
have the attribute specified by `field` with one of the values specified in
`filters`.
Return the id and count of the top `n` rows (by count of bits) in the field.
The `attrName` and `attrValues` arguments work together to only return rows which
have the attribute specified by `attrName` with one of the values specified in
`attrValues`.
**Result Type:** array of key/count objects
**Caveats:**
* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order.
* Frames with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of frame will return bitmaps sorted in order of most recently set bit.
* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
* Once full, the cache will truncate the set of bitmaps according to the frame option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order.
* The TopN query's attribute filter is applied to the existing sorted cache of bitmaps. Bitmaps that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order.
* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit.
* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
* Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order.
* The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored.
See [frame creation](../api-reference/#create-frame) for more information about the cache.
See [field creation](../api-reference/#create-field) for more information about the cache.
**Examples:**
Basic TopN query:
```request
TopN(frame="stargazer")
TopN(stargazer)
```
```response
{"results":[[{"id":1240,"count":102},{"id":4734,"count":100},{"id":12709,"count":93},...]]}
@ -479,11 +454,11 @@ TopN(frame="stargazer")
* `id` is a row ID (user ID)
* `count` is a count of columns (repositories)
* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer frame. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository.
* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer field. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository.
Limit the number of results:
```request
TopN(frame="stargazer", n=2)
TopN(stargazer, n=2)
```
```response
{"results":[[{"id":1240,"count":102},{"id":4734,"count":100}]]}
@ -491,19 +466,19 @@ TopN(frame="stargazer", n=2)
* Results are the top two rows (users) sorted by number of bits set (repositories they've starred) in descending order.
Filter based on an existing Bitmap:
Filter based on an existing row:
```request
TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2)
TopN(Row(language=1), stargazer, n=2)
```
```response
{"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]}
```
* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language frame (repositories that they've starred which are written in language 1).
* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language field (repositories that they've starred which are written in language 1).
Filter based on attributes:
```request
TopN(frame="stargazer", n=2, field=active, filters=[true])
TopN(stargazer, n=2, attrName=active, attrValues=[true])
```
```response
{"results":[[{"id":10,"count":1},{"id":13,"count":1}]]}
@ -516,31 +491,30 @@ TopN(frame="stargazer", n=2, field=active, filters=[true])
**Spec:**
```
Range(<frame=STRING>, <row=UINT>,
<start=TIMESTAMP>, <end=TIMESTAMP>)
Range(<FIELD>=<ROW>, <TIMESTAMP>, <TIMESTAMP>)
```
**Description:**
Similar to `Bitmap`, but only returns bits which were set with timestamps
between the given `start` and `end` timestamps.
Similar to `Row`, but only returns bits which were set with timestamps
between the given `start` (first) and `end` (second) timestamps.
**Result Type:** object with attrs and bits
**Examples:**
Query all columns with a bit set in row 1 of a frame (repositories that a user has starred), within a date range:
Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
```request
Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00)
```
```response
{{"attrs":{},"bits":[10]}
{{"attrs":{},"columns":[10]}
```
This example assumes timestamps have been set on some bits.
* bits are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
#### Range (BSI)
@ -548,16 +522,15 @@ This example assumes timestamps have been set on some bits.
**Spec:**
```
Range(<frame=STRING>, <FIELD_NAME, COMPARISON_OPERATOR, COMPARISON_VALUE> )
Range([<COMPARISON_VALUE> <COMPARISON_OPERATOR>] <FIELD> <COMPARISON_OPERATOR> <COMPARISON_VALUE>)
```
**Description:**
The `Range` query is overloaded to work on `field` values as well as `timestamp` values.
The `Range` query is overloaded to work on `integer` values as well as `timestamp` values.
Returns bits that are true for the comparison operator.
**Result Type:** object with attrs and bits
**Result Type:** object with attrs and columns
**Examples:**
@ -565,13 +538,13 @@ In our source data, commitactivity was counted over the last year.
The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits):
```request
Range(frame="stats", commitactivity > 100)
Range(commitactivity > 100)
```
```response
{{"attrs":{},"bits":[10]}
{{"attrs":{},"columns":[10]}
```
* bits are repositories which had at least 100 commits in the last year.
* columns are repositories which had at least 100 commits in the last year.
BSI range queries support the following operators:
@ -583,35 +556,37 @@ BSI range queries support the following operators:
`>=` | greater-than-or-equal-to, GTE | integer
`==` | equal-to, EQ | integer
`!=` | not-equal-to, NEQ | integer or `null`
`><` | between, BETWEEN | [integer, integer]
The `BETWEEN` form specifies an interval with both bounds, using the `><` operator, and a two-element list containing the lower and upper bounds of the interval:
`<`, and `<=` can be chained together to represent a bounded interval. For example:
```pql
Range(frame="stats", commitactivity >< [100, 200])
```request
Range(50 < commitactivity < 150)
```
```response
{{"attrs":{},"columns":[10]}
```
This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way.
As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported.
#### Min
**Spec:**
```
Min([BITMAP_CALL], <frame=STRING>, <field=STRING>)
Min([ROW_CALL], field=<FIELD>)
```
**Description:**
Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
Returns the minimum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
**Result Type:** object with the min and count of columns containing the min value.
**Examples:**
Query the minimum value of all fields in a frame (minimum size of all repositories):
Query the minimum value of a field (minimum size of all repositories):
```request
Min(frame="stats", field="diskusage")
Min(field="diskusage")
```
```response
{"value":4,"count":2}
@ -624,20 +599,20 @@ Min(frame="stats", field="diskusage")
**Spec:**
```
Max([BITMAP_CALL], <frame=STRING>, <field=STRING>)
Max([ROW_CALL], field=<FIELD>)
```
**Description:**
Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
Returns the maximum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered.
**Result Type:** object with the max and count of columns containing the max value.
**Examples:**
Query the maximum value of all fields in a frame (maximum size of all repositories):
Query the maximum value of a field (maximum size of all repositories):
```request
Max(frame="stats", field="diskusage")
Max(field="diskusage")
```
```response
{"value":88,"count":13}
@ -650,20 +625,20 @@ Max(frame="stats", field="diskusage")
**Spec:**
```
Sum([BITMAP_CALL], <frame=STRING>, <field=STRING>)
Sum([ROW_CALL], field=<FIELD>)
```
**Description:**
Returns the count and computed sum of all BSI integer values in the `field` and `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
**Result Type:** object with the computed sum and count of the bitmap field.
**Result Type:** object with the computed sum and count of the values in the integer field.
**Examples:**
Query the size of all repositories.
```request
Sum(frame="stats", field="diskusage")
Sum(field="diskusage")
```
```response
{"value":10,"count":3}

166
docs/tutorials.md
View file

@ -407,37 +407,65 @@ curl localhost:10101/index/patients \
-X POST
```
``` response
{}
{"success":true}
```
In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
In addition to storing rows of bits, a frame can also contain fields that store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
``` request
curl localhost:10101/index/patients/frame/measurements \
curl localhost:10101/index/patients/field/age \
-X POST \
-d '{"options":{
"fields": [
{"name": "age", "type": "int", "min": 0, "max": 120},
{"name": "weight", "type": "int", "min": 0, "max": 500},
{"name": "tcells", "type": "int", "min": 0, "max": 2000}
]
}}'
-d '{"options":{"type": "int", "min": 0, "max": 120}}'
```
``` response
{}
{"success":true}
```
If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field).
``` request
curl localhost:10101/index/patients/field/weight \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 500}}'
```
``` response
{"success":true}
```
``` request
curl localhost:10101/index/patients/field/tcells \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 2000}}'
```
``` response
{"success":true}
```
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
This query sets the age, weight, and t-cell count for the patient with ID `1` in our system:
The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)'
-d 'Set(1, age=34)'
```
``` response
{"results":[null]}
{"results":[true]}
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Set(1, weight=128)'
```
``` response
{"results":[true]}
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Set(1, tcells=1145)'
```
``` response
{"results":[true]}
```
In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
@ -454,7 +482,7 @@ Assuming we have a file called `ages.csv` that is structured like this:
8,33
9,63
```
where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command:
where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command:
```
pilosa import -i patients -f measurements --field age ages.csv
```
@ -465,10 +493,10 @@ In order to find all patients over the age of 40, then simply run a `Range` quer
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Range(frame="measurements", age > 40)'
-d 'Range(age > 40)'
```
``` response
{"results":[{"attrs":{},"bits":[2,6,9]}]}
{"results":[{"attrs":{},"columns":[2,6,9]}]}
```
You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
@ -477,21 +505,21 @@ To find the average age of all patients, run a `Sum` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(frame="measurements", field="age")'
-d 'Sum(field="age")'
```
``` response
{"results":[{"sum":377,"count":9}]}
{"results":[{"value":377,"count":9}]}
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`.
You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")'
-d 'Sum(Range(age > 40), field="age")'
```
``` response
{"results":[{"sum":191,"count":3}]}
{"results":[{"value":191,"count":3}]}
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
@ -499,42 +527,42 @@ To find the minimum age of all patients, run a `Min` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(frame="measurements", field="age")'
-d 'Min(field="age")'
```
``` response
{"results":[{"min":19,"count":1}]}
{"results":[{"value":19,"count":1}]}
```
The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value.
The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")'
-d 'Min(Range(age > 40), field="age")'
```
``` response
{"results":[{"min":57,"count":1}]}
{"results":[{"value":57,"count":1}]}
```
To find the maximum age of all patients, run a `Max` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(frame="measurements", field="age")'
-d 'Max(field="age")'
```
``` response
{"results":[{"max":71,"count":1}]}
{"results":[{"value":71,"count":1}]}
```
The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value.
The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")'
-d 'Max(Range(age < 40), field="age")'
```
``` response
{"results":[{"max":34,"count":1}]}
{"results":[{"value":34,"count":1}]}
```
### Storing Row and Column Attributes
@ -549,28 +577,28 @@ curl localhost:10101/index/books \
-X POST
```
``` response
{}
{"success":true}
```
Next, create a frame in the `books` index called `members` which will represent library members who have read books.
Next, create a field in the `books` index called `members` which will represent library members who have read books.
``` request
curl localhost:10101/index/books/frame/members \
curl localhost:10101/index/books/field/members \
-X POST \
-d '{}'
```
``` response
{}
{"success":true}
```
Now, let's add some books to our index.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(col=2, name="No Name in the Street", year=1972)
SetColumnAttrs(col=3, name="The Tipping Point", year=2000)
SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(col=5, name="The Forever War", year=2008)'
-d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(2, name="No Name in the Street", year=1972)
SetColumnAttrs(3, name="The Tipping Point", year=2000)
SetColumnAttrs(4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(5, name="The Forever War", year=2008)'
```
``` response
{"results":[null,null,null,null,null]}
@ -580,11 +608,11 @@ And add some members.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith")
SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")'
-d 'SetRowAttrs(members, 10001, fullName="John Smith")
SetRowAttrs(members, 10002, fullName="Sue Perkins")
SetRowAttrs(members, 10003, fullName="Jennifer Hawks")
SetRowAttrs(members, 10004, fullName="Pedro Vazquez")
SetRowAttrs(members, 10005, fullName="Pat Washington")'
```
``` response
{"results":[null,null,null,null,null]}
@ -594,29 +622,29 @@ At this point we can query one of the `member` records by querying that row.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", row=10002)'
-d 'Row(members=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]}
```
Now let's add some data to the matrix such that each pair represents a member who has read that book.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetBit(frame="members", row=10001, col=3)
SetBit(frame="members", row=10001, col=5)
SetBit(frame="members", row=10002, col=1)
SetBit(frame="members", row=10002, col=2)
SetBit(frame="members", row=10002, col=4)
SetBit(frame="members", row=10003, col=3)
SetBit(frame="members", row=10004, col=4)
SetBit(frame="members", row=10004, col=5)
SetBit(frame="members", row=10005, col=1)
SetBit(frame="members", row=10005, col=2)
SetBit(frame="members", row=10005, col=3)
SetBit(frame="members", row=10005, col=4)
SetBit(frame="members", row=10005, col=5)'
-d 'Set(3, members=10001)
Set(5, members=10001)
Set(1, members=10002)
Set(2, members=10002)
Set(4, members=10002)
Set(3, members=10003)
Set(4, members=10004)
Set(5, members=10004)
Set(1, members=10005)
Set(2, members=10005)
Set(3, members=10005)
Set(4, members=10005)
Set(5, members=10005)'
```
``` response
{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
@ -626,22 +654,22 @@ Now pull the record for `Sue Perkins` again.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", row=10002)'
-d 'Row(members=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]}
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read.
In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="members", row=10002)'
-d 'Row(members=10002)'
```
``` response
{
"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}],
"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}],
"columnAttrs":[
{"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}},
{"id":2,"attrs":{"name":"No Name in the Street","year":1972}},
@ -655,11 +683,11 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`,
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))'
-d 'Intersect(Row(members=10002), Row(members=10004))'
```
``` response
{
"results":[{"attrs":{},"bits":[4]}],
"results":[{"attrs":{},"columns":[4]}],
"columnAttrs":[
{"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
]

10
docs/webui.md
View file

@ -43,14 +43,14 @@ In addition to standard PQL, the console supports a few special commands, prefix
- `:create index <indexname>`
- `:delete index <indexname>`
- `:use <indexname>`
- `:create frame <framename>`
- `:delete frame <framename>`
- `:create field <fieldname>`
- `:delete field <fieldname>`
Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field).
- `:create frame <framename> cacheSize=10000`
- `:create field <fieldname> cacheSize=10000`
### Cluster Admin
Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.
Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields.

1037
encoding/proto/proto.go Normal file
View file

File diff suppressed because it is too large Load diff

88
enterprise/b/btree.go
View file

@ -56,23 +56,23 @@ func init() {
var (
btDPool = sync.Pool{New: func() interface{} { return &d{} }}
btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}}
btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}}
btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}}
btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}}
btXPool = sync.Pool{New: func() interface{} { return &x{} }}
)
type btTpool struct{ sync.Pool }
func (p *btTpool) get(cmp Cmp) *Tree {
x := p.Get().(*Tree)
func (p *btTpool) get(cmp Cmp) *tree {
x := p.Get().(*tree)
x.cmp = cmp
return x
}
type btEpool struct{ sync.Pool }
func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator {
x := p.Get().(*Enumerator)
func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator {
x := p.Get().(*enumerator)
x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver
return x
}
@ -98,26 +98,26 @@ type (
v *roaring.Container
}
// Enumerator captures the state of enumerating a tree. It is returned
// enumerator captures the state of enumerating a tree. It is returned
// from the Seek* methods. The enumerator is aware of any mutations
// made to the tree in the process of enumerating it and automatically
// resumes the enumeration at the proper key, if possible.
//
// However, once an Enumerator returns io.EOF to signal "no more
// However, once an enumerator returns io.EOF to signal "no more
// items", it does no more attempt to "resync" on tree mutation(s). In
// other words, io.EOF from an Enumerator is "sticky" (idempotent).
Enumerator struct {
// other words, io.EOF from an enumerator is "sticky" (idempotent).
enumerator struct {
err error
hit bool
i int
k uint64
q *d
t *Tree
t *tree
ver int64
}
// Tree is a B+tree.
Tree struct {
// tree is a B+tree.
tree struct {
c int
cmp Cmp
first *d
@ -140,9 +140,9 @@ type (
var ( // R/O zero values
zd d
zde de
ze Enumerator
ze enumerator
zk uint64
zt Tree
zt tree
zx x
zxe xe
)
@ -233,14 +233,14 @@ func (l *d) mvR(r *d, c int) {
// ----------------------------------------------------------------------- Tree
// TreeNew returns a newly created, empty Tree. The compare function is used
// treeNew returns a newly created, empty Tree. The compare function is used
// for key collation.
func TreeNew(cmp Cmp) *Tree {
func treeNew(cmp Cmp) *tree {
return btTPool.get(cmp)
}
// Clear removes all K/V pairs from the tree.
func (t *Tree) Clear() {
func (t *tree) Clear() {
if t.r == nil {
return
}
@ -252,13 +252,13 @@ func (t *Tree) Clear() {
// Close performs Clear and recycles t to a pool for possible later reuse. No
// references to t should exist or such references must not be used afterwards.
func (t *Tree) Close() {
func (t *tree) Close() {
t.Clear()
*t = zt
btTPool.Put(t)
}
func (t *Tree) cat(p *x, q, r *d, pi int) {
func (t *tree) cat(p *x, q, r *d, pi int) {
t.ver++
q.mvL(r, r.c)
if r.n != nil {
@ -286,7 +286,7 @@ func (t *Tree) cat(p *x, q, r *d, pi int) {
t.r = q
}
func (t *Tree) catX(p, q, r *x, pi int) {
func (t *tree) catX(p, q, r *x, pi int) {
t.ver++
q.x[q.c].k = p.x[pi].k
copy(q.x[q.c+1:], r.x[:r.c])
@ -320,7 +320,7 @@ func (t *Tree) catX(p, q, r *x, pi int) {
// Delete removes the k's KV pair, if it exists, in which case Delete returns
// true.
func (t *Tree) Delete(k uint64) (ok bool) {
func (t *tree) Delete(k uint64) (ok bool) {
pi := -1
var p *x
q := t.r
@ -370,7 +370,7 @@ func (t *Tree) Delete(k uint64) (ok bool) {
}
}
func (t *Tree) extract(q *d, i int) { // (r *container) {
func (t *tree) extract(q *d, i int) { // (r *container) {
t.ver++
//r = q.d[i].v // prepared for Extract
q.c--
@ -381,7 +381,7 @@ func (t *Tree) extract(q *d, i int) { // (r *container) {
t.c--
}
func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) {
func (t *tree) find(q interface{}, k uint64) (i int, ok bool) {
var mk uint64
l := 0
switch x := q.(type) {
@ -419,7 +419,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) {
// First returns the first item of the tree in the key collating order, or
// (zero-value, zero-value) if the tree is empty.
func (t *Tree) First() (k uint64, v *roaring.Container) {
func (t *tree) First() (k uint64, v *roaring.Container) {
if q := t.first; q != nil {
q := &q.d[0]
k, v = q.k, q.v
@ -429,7 +429,7 @@ func (t *Tree) First() (k uint64, v *roaring.Container) {
// Get returns the value associated with k and true if it exists. Otherwise Get
// returns (zero-value, false).
func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) {
func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) {
q := t.r
if q == nil {
return
@ -455,7 +455,7 @@ func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) {
}
}
func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
t.ver++
c := q.c
if i < c {
@ -470,7 +470,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d {
// Last returns the last item of the tree in the key collating order, or
// (zero-value, zero-value) if the tree is empty.
func (t *Tree) Last() (k uint64, v *roaring.Container) {
func (t *tree) Last() (k uint64, v *roaring.Container) {
if q := t.last; q != nil {
q := &q.d[q.c-1]
k, v = q.k, q.v
@ -479,11 +479,11 @@ func (t *Tree) Last() (k uint64, v *roaring.Container) {
}
// Len returns the number of items in the tree.
func (t *Tree) Len() int {
func (t *tree) Len() int {
return t.c
}
func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
t.ver++
l, r := p.siblings(pi)
@ -528,7 +528,7 @@ func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
// Seek returns an Enumerator positioned on an item such that k >= item's key.
// ok reports if k == item.key The Enumerator's position is possibly after the
// last item in the tree.
func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) {
func (t *tree) Seek(k uint64) (e *enumerator, ok bool) {
q := t.r
if q == nil {
e = btEPool.get(nil, false, 0, k, nil, t, t.ver)
@ -558,7 +558,7 @@ func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) {
// SeekFirst returns an enumerator positioned on the first KV pair in the tree,
// if any. For an empty tree, err == io.EOF is returned and e will be nil.
func (t *Tree) SeekFirst() (e *Enumerator, err error) {
func (t *tree) SeekFirst() (e *enumerator, err error) {
q := t.first
if q == nil {
return nil, io.EOF
@ -569,7 +569,7 @@ func (t *Tree) SeekFirst() (e *Enumerator, err error) {
// SeekLast returns an enumerator positioned on the last KV pair in the tree,
// if any. For an empty tree, err == io.EOF is returned and e will be nil.
func (t *Tree) SeekLast() (e *Enumerator, err error) {
func (t *tree) SeekLast() (e *enumerator, err error) {
q := t.last
if q == nil {
return nil, io.EOF
@ -579,7 +579,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) {
}
// Set sets the value associated with k.
func (t *Tree) Set(k uint64, v *roaring.Container) {
func (t *tree) Set(k uint64, v *roaring.Container) {
//dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump())
//defer func() {
// dbg("--- POST\n%s\n====\n", t.dump())
@ -645,7 +645,7 @@ func (t *Tree) Set(k uint64, v *roaring.Container) {
// tree.Put(k, func(uint64, bool){ return v, true })
//
// modulo the differing return values.
func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) {
func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) {
pi := -1
var p *x
q := t.r
@ -712,7 +712,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (new
}
}
func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
t.ver++
r := btDPool.Get().(*d)
if q.n != nil {
@ -747,7 +747,7 @@ func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) {
t.insert(q, i, k, v)
}
func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
t.ver++
r := btXPool.Get().(*x)
copy(r.x[:], q.x[kx+1:])
@ -771,7 +771,7 @@ func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) {
return q, i
}
func (t *Tree) underflow(p *x, q *d, pi int) {
func (t *tree) underflow(p *x, q *d, pi int) {
t.ver++
l, r := p.siblings(pi)
@ -796,7 +796,7 @@ func (t *Tree) underflow(p *x, q *d, pi int) {
t.cat(p, q, r, pi)
}
func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
t.ver++
var l, r *x
@ -850,7 +850,7 @@ func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) {
// Close recycles e to a pool for possible later reuse. No references to e
// should exist or such references must not be used afterwards.
func (e *Enumerator) Close() {
func (e *enumerator) Close() {
*e = ze
btEPool.Put(e)
}
@ -858,7 +858,7 @@ func (e *Enumerator) Close() {
// Next returns the currently enumerated item, if it exists and moves to the
// next item in the key collation order. If there is no item to return, err ==
// io.EOF is returned.
func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) {
func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) {
if err = e.err; err != nil {
return
}
@ -886,7 +886,7 @@ func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) {
return
}
func (e *Enumerator) next() error {
func (e *enumerator) next() error {
if e.q == nil {
e.err = io.EOF
return io.EOF
@ -906,7 +906,7 @@ func (e *Enumerator) next() error {
// Prev returns the currently enumerated item, if it exists and moves to the
// previous item in the key collation order. If there is no item to return, err
// == io.EOF is returned.
func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) {
func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) {
if err = e.err; err != nil {
return
}
@ -941,7 +941,7 @@ func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) {
return
}
func (e *Enumerator) prev() error {
func (e *enumerator) prev() error {
if e.q == nil {
e.err = io.EOF
return io.EOF

48
enterprise/b/containers_btree.go
View file

@ -27,28 +27,28 @@ func cmp(a, b uint64) int {
return int(a - b)
}
type BTreeContainers struct {
tree *Tree
type bTreeContainers struct {
tree *tree
lastKey uint64
lastContainer *roaring.Container
}
func NewBTreeContainers() *BTreeContainers {
return &BTreeContainers{
tree: TreeNew(cmp),
func newBTreeContainers() *bTreeContainers {
return &bTreeContainers{
tree: treeNew(cmp),
}
}
func NewBTreeBitmap(a ...uint64) *roaring.Bitmap {
b := &roaring.Bitmap{
Containers: NewBTreeContainers(),
Containers: newBTreeContainers(),
}
b.Add(a...)
return b
}
func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
func (btc *bTreeContainers) Get(key uint64) *roaring.Container {
// Check the last* cache for same container.
if key == btc.lastKey && btc.lastContainer != nil {
return btc.lastContainer
@ -64,7 +64,7 @@ func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
return c
}
func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) {
func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) {
// If a mapped container is added to the tree, reset the
// lastContainer cache so that the cache is not pointing
// at a read-only mmap.
@ -93,16 +93,16 @@ type updater struct {
mapped bool
}
func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
a := updater{key, containerType, n, mapped}
btc.tree.Put(key, a.update)
}
func (btc *BTreeContainers) Remove(key uint64) {
func (btc *bTreeContainers) Remove(key uint64) {
btc.tree.Delete(key)
}
func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container {
// Check the last* cache for same container.
if key == btc.lastKey && btc.lastContainer != nil {
return btc.lastContainer
@ -121,7 +121,7 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
return btc.lastContainer
}
func (btc *BTreeContainers) Count() (n uint64) {
func (btc *bTreeContainers) Count() (n uint64) {
e, _ := btc.tree.Seek(0)
_, c, err := e.Next()
for err != io.EOF {
@ -131,8 +131,8 @@ func (btc *BTreeContainers) Count() (n uint64) {
return
}
func (btc *BTreeContainers) Clone() roaring.Containers {
nbtc := NewBTreeContainers()
func (btc *bTreeContainers) Clone() roaring.Containers {
nbtc := newBTreeContainers()
itr, err := btc.tree.SeekFirst()
if err == io.EOF {
@ -148,7 +148,7 @@ func (btc *BTreeContainers) Clone() roaring.Containers {
return nbtc
}
func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) {
if btc.tree.Len() == 0 {
return 0, nil
}
@ -156,34 +156,34 @@ func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
return k, v
}
func (btc *BTreeContainers) Size() int {
func (btc *bTreeContainers) Size() int {
return btc.tree.Len()
}
func (btc *BTreeContainers) Reset() {
btc.tree = TreeNew(cmp)
func (btc *bTreeContainers) Reset() {
btc.tree = treeNew(cmp)
btc.lastKey = 0
btc.lastContainer = nil
}
func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
e, ok := btc.tree.Seek(key)
if ok {
found = true
}
return &BTCIterator{
return &btcIterator{
e: e,
}, found
}
type BTCIterator struct {
e *Enumerator
type btcIterator struct {
e *enumerator
key uint64
val *roaring.Container
}
func (i *BTCIterator) Next() bool {
func (i *btcIterator) Next() bool {
k, v, err := i.e.Next()
if err == io.EOF {
@ -194,7 +194,7 @@ func (i *BTCIterator) Next() bool {
return true
}
func (i *BTCIterator) Value() (uint64, *roaring.Container) {
func (i *btcIterator) Value() (uint64, *roaring.Container) {
if i.val == nil {
return 0, nil
}

4
event.go
View file

@ -23,8 +23,8 @@ const (
NodeUpdate
)
// nodeEvent is a single event related to node activity in the cluster.
type nodeEvent struct {
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
}

225
executor.go
View file

@ -20,7 +20,6 @@ import (
"sort"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -68,7 +67,7 @@ func optExecutorInternalQueryClient(c InternalQueryClient) executorOption {
// newExecutor returns a new instance of Executor.
func newExecutor(opts ...executorOption) *executor {
e := &executor{
client: NewNopInternalQueryClient(),
client: newNopInternalQueryClient(),
}
for _, opt := range opts {
err := opt(e)
@ -184,9 +183,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, shards, opt)
case "Set":
return e.executeSetBit(ctx, index, c, opt)
case "SetValue":
return nil, e.executeSetValue(ctx, index, c, opt)
return e.executeSet(ctx, index, c, opt)
case "SetRowAttrs":
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
@ -237,7 +234,7 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Add(v.(ValCount))
return other.add(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
@ -270,7 +267,7 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Smaller(v.(ValCount))
return other.smaller(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
@ -303,7 +300,7 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.Larger(v.(ValCount))
return other.larger(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
@ -337,7 +334,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "map reduce")
}
// Attach attributes for Row() calls.
@ -378,7 +375,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
}
if opt.ExcludeColumns {
row.segments = []RowSegment{}
row.segments = []rowSegment{}
}
return row, nil
@ -664,7 +661,7 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *
other = other.Difference(row)
}
}
other.InvalidateCount()
other.invalidateCount()
return other, nil
}
@ -715,10 +712,10 @@ func (e *executor) executeIntersectShard(ctx context.Context, index string, c *p
if i == 0 {
other = row
} else {
other = other.Intersect(row)
other = other.intersect(row)
}
}
other.InvalidateCount()
other.invalidateCount()
return other, nil
}
@ -940,7 +937,7 @@ func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.C
other = other.Union(row)
}
}
other.InvalidateCount()
other.invalidateCount()
return other, nil
}
@ -959,7 +956,7 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal
other = other.Xor(row)
}
}
other.InvalidateCount()
other.invalidateCount()
return other, nil
}
@ -1060,8 +1057,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
return ret, nil
}
// executeSetBit executes a Set() call.
func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Set() argument required: field")
@ -1077,14 +1074,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
// Read colID using labels.
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading Set() column: %v", err)
@ -1092,20 +1082,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, fmt.Errorf("Set() column argument '%v' required", columnLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if f.Type() == FieldTypeInt {
// Read remaining fields using labels.
rowVal, ok, err := c.IntArg(fieldName)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
} else {
// Read remaining fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
}
// executeSetBitField executes a Set() call for a specific view.
// executeSetBitField executes a Set() call for a specific field.
func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) {
shard := colID / ShardWidth
ret := false
@ -1137,64 +1147,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
return ret, nil
}
// executeSetValue executes a SetValue() call.
func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error {
// Parse labels.
columnID, ok, err := c.UintArg(columnLabel)
if err != nil {
return fmt.Errorf("reading SetValue() column: %v", err)
} else if !ok {
return fmt.Errorf("SetValue() column field '%v' required", columnLabel)
}
// executeSetValueField executes a Set() call for a specific int field.
func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) {
shard := colID / ShardWidth
ret := false
// Copy args and remove reserved fields.
args := pql.CopyArgs(c.Args)
// While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion.
// Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility.
delete(args, columnLabel)
// Set values.
for name, value := range args {
// Retrieve field.
field := e.Holder.Field(index, name)
if field == nil {
return ErrFieldNotFound
}
switch value := value.(type) {
case int64:
if _, err := field.SetValue(columnID, value); err != nil {
return err
for _, node := range e.Cluster.shardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetValue(colID, value)
if err != nil {
return false, err
} else if val {
ret = true
}
default:
return ErrInvalidBSIGroupValueType
continue
}
field.Stats.Count("SetValue", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
return nil
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
continue
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
// Return first error.
for range nodes {
if err := <-resp; err != nil {
return err
// Forward call to remote node otherwise.
if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
}
}
return nil
return ret, nil
}
// executeSetRowAttrs executes a SetRowAttrs() call.
@ -1392,7 +1374,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
// exec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
pbreq := &QueryRequest{
Query: q.String(),
Shards: shards,
Remote: true,
@ -1403,40 +1385,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *
return nil, err
}
// Return an error, if specified on response.
if err := decodeError(pb.Err); err != nil {
return nil, err
}
// Return appropriate data for the query.
results = make([]interface{}, len(q.Calls))
for i, call := range q.Calls {
var v interface{}
var err error
switch call.Name {
case "Average", "Sum":
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
v, err = pb.Results[i].N, nil
case "Set":
v, err = pb.Results[i].Changed, nil
case "Clear":
v, err = pb.Results[i].Changed, nil
case "SetRowAttrs":
case "SetColumnAttrs":
default:
v, err = DecodeRow(pb.Results[i].GetRow()), nil
}
if err != nil {
return nil, err
}
results[i] = v
}
return results, nil
return pb.Results, pb.Err
}
// shardsByNode returns a mapping of nodes to shards.
@ -1490,7 +1439,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
return nil, errors.Wrap(ctx.Err(), "context done")
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
@ -1500,10 +1449,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable {
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, err
return nil, errors.Wrap(err, "calling mapper")
}
continue
}
@ -1524,7 +1473,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return err
return errors.Wrap(err, "shards by node")
}
// Execute each node in a separate goroutine.
@ -1628,7 +1577,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
if field == nil {
return ErrFieldNotFound
}
if field.Keys() {
if field.keys() {
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
return errors.New("row value must be a string when field 'keys' option enabled")
}
@ -1679,7 +1628,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
if field == nil {
return nil, ErrFieldNotFound
}
if field.Keys() {
if field.keys() {
other := make([]Pair, len(result))
for i := range result {
key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID)
@ -1764,29 +1713,15 @@ type ValCount struct {
Count int64 `json:"count"`
}
func (vc *ValCount) Add(other ValCount) ValCount {
func (vc *ValCount) add(other ValCount) ValCount {
return ValCount{
Val: vc.Val + other.Val,
Count: vc.Count + other.Count,
}
}
func EncodeValCount(vc ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
func decodeValCount(pb *internal.ValCount) ValCount {
return ValCount{
Val: pb.Val,
Count: pb.Count,
}
}
// Smaller returns the smaller of the two ValCounts.
func (vc *ValCount) Smaller(other ValCount) ValCount {
// smaller returns the smaller of the two ValCounts.
func (vc *ValCount) smaller(other ValCount) ValCount {
if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {
return other
}
@ -1796,8 +1731,8 @@ func (vc *ValCount) Smaller(other ValCount) ValCount {
}
}
// Larger returns the larger of the two ValCounts.
func (vc *ValCount) Larger(other ValCount) ValCount {
// larger returns the larger of the two ValCounts.
func (vc *ValCount) larger(other ValCount) ValCount {
if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) {
return other
}

60
executor_test.go
View file

@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) {
hldr.SetBit("i", "f", 1, 0)
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" {
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err))
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err)
}
}
@ -405,9 +405,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Set bsiGroup values.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil {
t.Fatal(err)
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil {
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil {
t.Fatal(err)
}
@ -440,19 +440,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` {
t.Fatalf("unexpected error: %s", err)
}
})
@ -748,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
Set(1, x=1)
Set(` + strconv.Itoa(ShardWidth+2) + `, x=2)
SetValue(col=0, f=20)
SetValue(col=1, f=-5)
SetValue(col=2, f=-5)
SetValue(col=3, f=10)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60)
Set(0, f=20)
Set(1, f=-5)
Set(2, f=-5)
Set(3, f=10)
Set(` + strconv.Itoa(ShardWidth) + `, f=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, f=40)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=60)
`}); err != nil {
t.Fatal(err)
}
@ -844,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) {
Set(0, x=0)
Set(` + strconv.Itoa(ShardWidth+1) + `, x=0)
SetValue(col=0, foo=20)
SetValue(col=0, bar=2000)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
SetValue(col=0, other=1000)
Set(0, foo=20)
Set(0, bar=2000)
Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
Set(0, other=1000)
`}); err != nil {
t.Fatal(err)
}
@ -959,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
Set(0, f=0)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=0)
SetValue(col=50, foo=20)
SetValue(col=50, bar=2000)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
SetValue(col=0, other=1000)
SetValue(col=0, edge=100)
SetValue(col=1, edge=-100)
Set(50, foo=20)
Set(50, bar=2000)
Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
Set(0, other=1000)
Set(0, edge=100)
Set(1, edge=-100)
`}); err != nil {
t.Fatal(err)
}

133
field.go
View file

@ -68,25 +68,25 @@ type Field struct {
Stats StatsClient
// Field options.
options fieldOptions
options FieldOptions
bsiGroups []*bsiGroup
Logger Logger
logger Logger
}
// FieldOption is a functional option type for pilosa.fieldOptions.
type FieldOption func(fo *fieldOptions) error
type FieldOption func(fo *FieldOptions) error
func OptFieldKeys() FieldOption {
return func(fo *fieldOptions) error {
return func(fo *FieldOptions) error {
fo.Keys = true
return nil
}
}
func OptFieldTypeDefault() FieldOption {
return func(fo *fieldOptions) error {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
@ -98,7 +98,7 @@ func OptFieldTypeDefault() FieldOption {
}
func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
return func(fo *fieldOptions) error {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
@ -110,7 +110,7 @@ func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
}
func OptFieldTypeInt(min, max int64) FieldOption {
return func(fo *fieldOptions) error {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
@ -125,7 +125,7 @@ func OptFieldTypeInt(min, max int64) FieldOption {
}
func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
return func(fo *fieldOptions) error {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
@ -146,7 +146,7 @@ func NewField(path, index, name string, opts FieldOption) (*Field, error) {
}
// Apply functional option.
fo := fieldOptions{}
fo := FieldOptions{}
err = opts(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
@ -166,7 +166,7 @@ func NewField(path, index, name string, opts FieldOption) (*Field, error) {
options: applyDefaultOptions(fo),
Logger: NopLogger,
logger: NopLogger,
}
return f, nil
}
@ -183,8 +183,8 @@ func (f *Field) Path() string { return f.path }
// RowAttrStore returns the attribute storage.
func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore }
// MaxShard returns the max shard in the field.
func (f *Field) MaxShard() uint64 {
// maxShard returns the max shard in the field.
func (f *Field) maxShard() uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
@ -233,7 +233,7 @@ func (f *Field) CacheSize() uint32 {
}
// Options returns all options for this field.
func (f *Field) Options() fieldOptions {
func (f *Field) Options() FieldOptions {
f.mu.RLock()
defer f.mu.RUnlock()
return f.options
@ -337,7 +337,7 @@ func (f *Field) loadMeta() error {
func (f *Field) saveMeta() error {
// Marshal metadata.
fo := f.options
buf, err := proto.Marshal(fo.Encode())
buf, err := proto.Marshal(fo.encode())
if err != nil {
return errors.Wrap(err, "marshaling")
}
@ -351,7 +351,7 @@ func (f *Field) saveMeta() error {
}
// applyOptions configures the field based on opt.
func (f *Field) applyOptions(opt fieldOptions) error {
func (f *Field) applyOptions(opt FieldOptions) error {
switch opt.Type {
case FieldTypeSet, "":
f.options.Type = FieldTypeSet
@ -396,7 +396,7 @@ func (f *Field) applyOptions(opt fieldOptions) error {
f.options.Max = 0
f.options.Keys = opt.Keys
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
if err := f.setTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
return errors.Wrap(err, "setting time quantum")
}
@ -428,8 +428,8 @@ func (f *Field) Close() error {
return nil
}
// Keys returns true if the field uses string keys.
func (f *Field) Keys() bool {
// keys returns true if the field uses string keys.
func (f *Field) keys() bool {
f.mu.RLock()
defer f.mu.RUnlock()
return f.options.Keys
@ -533,8 +533,8 @@ func (f *Field) TimeQuantum() TimeQuantum {
return f.options.TimeQuantum
}
// SetTimeQuantum sets the time quantum for the field.
func (f *Field) SetTimeQuantum(q TimeQuantum) error {
// setTimeQuantum sets the time quantum for the field.
func (f *Field) setTimeQuantum(q TimeQuantum) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -606,8 +606,8 @@ func (f *Field) viewNames() []string {
return other
}
// RecalculateCaches recalculates caches on every view in the field.
func (f *Field) RecalculateCaches() {
// recalculateCaches recalculates caches on every view in the field.
func (f *Field) recalculateCaches() {
for _, view := range f.views() {
view.recalculateCaches()
}
@ -624,7 +624,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) {
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&internal.CreateViewMessage{
&CreateViewMessage{
Index: f.index,
Field: f.name,
View: name,
@ -660,7 +660,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
func (f *Field) newView(path, name string) *view {
view := newView(path, f.index, f.name, name, f.options.CacheSize)
view.cacheType = f.options.CacheType
view.logger = f.Logger
view.logger = f.logger
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name))
view.broadcaster = f.broadcaster
@ -955,7 +955,7 @@ func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error)
return view.rangeOp(op, bsig.BitDepth(), baseValue)
}
func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) {
func (f *Field) rangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) {
// Retrieve and validate bsiGroup.
bsig := f.bsiGroup(name)
if bsig == nil {
@ -1035,8 +1035,8 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
return nil
}
// ImportValue bulk imports range-encoded value data.
func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
// importValue bulk imports range-encoded value data.
func (f *Field) importValue(columnIDs []uint64, values []int64) error {
viewName := viewBSIGroupPrefix + f.name
// Get the bsiGroup so we know bitDepth.
bsig := f.bsiGroup(f.name)
@ -1092,40 +1092,6 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
return nil
}
func (f *Field) MarshalJSON() ([]byte, error) {
thing := struct {
Name string
Options fieldOptions
Views []*viewInfo
}{
Name: f.Name(),
Options: f.Options(),
}
for _, viewname := range f.viewNames() {
thing.Views = append(thing.Views, &viewInfo{Name: viewname})
}
return json.Marshal(thing)
}
// encodeFields converts a into its internal representation.
func encodeFields(a []*Field) []*internal.Field {
other := make([]*internal.Field, len(a))
for i := range a {
other[i] = encodeField(a[i])
}
return other
}
// encodeField converts f into its internal representation.
func encodeField(f *Field) *internal.Field {
fo := f.options
return &internal.Field{
Name: f.name,
Meta: fo.Encode(),
Views: f.viewNames(),
}
}
type fieldSlice []*Field
func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1135,8 +1101,8 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FieldInfo represents schema information for a field.
type FieldInfo struct {
Name string `json:"name"`
Options fieldOptions `json:"options"`
Views []*viewInfo `json:"views,omitempty"`
Options FieldOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type fieldInfoSlice []*FieldInfo
@ -1145,22 +1111,22 @@ func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p fieldInfoSlice) Len() int { return len(p) }
func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// fieldOptions represents options to set when initializing a field.
type fieldOptions struct {
// FieldOptions represents options to set when initializing a field.
type FieldOptions struct {
Type string `json:"type,omitempty"`
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Keys bool `json:"keys,omitempty"`
Keys bool `json:"keys"`
}
// applyDefaultOptions returns a new fieldOptions object
// applyDefaultOptions returns a new FieldOptions object
// with default values if o does not contain a valid type.
func applyDefaultOptions(o fieldOptions) fieldOptions {
func applyDefaultOptions(o FieldOptions) FieldOptions {
if o.Type == "" {
return fieldOptions{
return FieldOptions{
Type: DefaultFieldType,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
@ -1169,12 +1135,12 @@ func applyDefaultOptions(o fieldOptions) fieldOptions {
return o
}
// Encode converts o into its internal representation.
func (o *fieldOptions) Encode() *internal.FieldOptions {
// encode converts o into its internal representation.
func (o *FieldOptions) encode() *internal.FieldOptions {
return encodeFieldOptions(o)
}
func encodeFieldOptions(o *fieldOptions) *internal.FieldOptions {
func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
if o == nil {
return nil
}
@ -1189,50 +1155,41 @@ func encodeFieldOptions(o *fieldOptions) *internal.FieldOptions {
}
}
func decodeFieldOptions(options *internal.FieldOptions) *fieldOptions {
if options == nil {
return nil
}
return &fieldOptions{
Type: options.Type,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
Min: options.Min,
Max: options.Max,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Keys: options.Keys,
}
}
func (o *fieldOptions) MarshalJSON() ([]byte, error) {
func (o *FieldOptions) MarshalJSON() ([]byte, error) {
switch o.Type {
case FieldTypeSet:
return json.Marshal(struct {
Type string `json:"type"`
CacheType string `json:"cacheType"`
CacheSize uint32 `json:"cacheSize"`
Keys bool `json:"keys"`
}{
o.Type,
o.CacheType,
o.CacheSize,
o.Keys,
})
case FieldTypeInt:
return json.Marshal(struct {
Type string `json:"type"`
Min int64 `json:"min"`
Max int64 `json:"max"`
Keys bool `json:"keys"`
}{
o.Type,
o.Min,
o.Max,
o.Keys,
})
case FieldTypeTime:
return json.Marshal(struct {
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
}{
o.Type,
o.TimeQuantum,
o.Keys,
})
}
return nil, errors.New("invalid field type")

4
field_internal_test.go
View file

@ -282,7 +282,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil {
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
@ -300,7 +300,7 @@ func TestField_RowTime(t *testing.T) {
f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
defer f.Close()
if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil {
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
}

28
fragment.go
View file

@ -343,13 +343,13 @@ func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac
// We Clone() data because otherwise row will contains pointers to containers in storage.
// This causes unexpected results when we cache the row and try to use it later.
row := &Row{
segments: []RowSegment{{
segments: []rowSegment{{
data: *data.Clone(),
shard: f.shard,
writable: false,
}},
}
row.InvalidateCount()
row.invalidateCount()
if updateRowCache {
f.rowCache.Add(rowID, row)
@ -446,7 +446,7 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
// Get the row from cache or fragment.storage.
row := f.unprotectedRow(rowID, true, true)
row.ClearBit(columnID)
row.clearBit(columnID)
// Update the cache.
f.cache.Add(rowID, row.Count())
@ -566,7 +566,7 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error
// Compute count based on the existence row.
row := f.row(uint64(bitDepth))
if filter != nil {
count = row.IntersectionCount(filter)
count = row.intersectionCount(filter)
} else {
count = row.Count()
}
@ -582,7 +582,7 @@ func (f *fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error
row := f.row(uint64(i))
cnt := uint64(0)
if filter != nil {
cnt = row.IntersectionCount(filter)
cnt = row.intersectionCount(filter)
} else {
cnt = row.Count()
}
@ -598,7 +598,7 @@ func (f *fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
consider = consider.intersect(filter)
}
// If there are no columns to consider, return early.
@ -631,7 +631,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
consider = consider.intersect(filter)
}
// If there are no columns to consider, return early.
@ -643,7 +643,7 @@ func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.row(uint64(ii))
x := row.Intersect(consider)
x := row.intersect(consider)
count = x.Count()
if count > 0 {
max += (1 << ii)
@ -682,7 +682,7 @@ func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
bit := (predicate >> uint(i)) & 1
if bit == 1 {
b = b.Intersect(row)
b = b.intersect(row)
} else {
b = b.Difference(row)
}
@ -783,7 +783,7 @@ func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool)
// If bit is unset then add columns with set bit to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(b.Intersect(row))
keep = keep.Union(b.intersect(row))
}
}
@ -815,7 +815,7 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64
// If bit is unset then add columns with set bit to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep1 = keep1.Union(b.Intersect(row))
keep1 = keep1.Union(b.intersect(row))
}
}
@ -938,7 +938,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) {
// Calculate count and append.
count := cnt
if opt.Src != nil {
count = opt.Src.IntersectionCount(f.row(rowID))
count = opt.Src.intersectionCount(f.row(rowID))
}
if count == 0 {
continue
@ -982,7 +982,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) {
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.row(rowID))
count := opt.Src.intersectionCount(f.row(rowID))
if count < threshold {
continue
}
@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
}
// Execute query.
queryRequest := &internal.QueryRequest{
queryRequest := &QueryRequest{
Query: buffers[k].String(),
Remote: true,
}

2
fragment_internal_test.go
View file

@ -1063,7 +1063,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
// Start benchmark
b.ResetTimer()
for i := 0; i < b.N; i++ {
if n := f.row(1).IntersectionCount(f.row(2)); n == 0 {
if n := f.row(1).intersectionCount(f.row(2)); n == 0 {
b.Fatalf("unexpected count: %d", n)
}
}

12
gcnotify/gcnotify.go
View file

@ -20,25 +20,25 @@ import (
)
// Ensure ActiveGCNotifier implements interface.
var _ pilosa.GCNotifier = &ActiveGCNotifier{}
var _ pilosa.GCNotifier = &activeGCNotifier{}
type ActiveGCNotifier struct {
type activeGCNotifier struct {
gcn *gcnotifier.GCNotifier
}
// NewActiveGCNotifier creates an active GCNotifier.
func NewActiveGCNotifier() *ActiveGCNotifier {
return &ActiveGCNotifier{
func NewActiveGCNotifier() *activeGCNotifier {
return &activeGCNotifier{
gcn: gcnotifier.New(),
}
}
// Close implements the GCNotifier interface.
func (n *ActiveGCNotifier) Close() {
func (n *activeGCNotifier) Close() {
n.gcn.Close()
}
// AfterGC implements the GCNotifier interface.
func (n *ActiveGCNotifier) AfterGC() <-chan struct{} {
func (n *activeGCNotifier) AfterGC() <-chan struct{} {
return n.gcn.AfterGC()
}

26
gopsutil/systeminfo.go
View file

@ -22,15 +22,15 @@ import (
var _ pilosa.SystemInfo = NewSystemInfo()
// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS.
type SystemInfo struct {
// systemInfo is an implementation of pilosa.systemInfo that uses gopsutil to collect information about the host OS.
type systemInfo struct {
platform string
family string
osVersion string
}
// Uptime returns the system uptime in seconds.
func (s *SystemInfo) Uptime() (uptime uint64, err error) {
func (s *systemInfo) Uptime() (uptime uint64, err error) {
hostInfo, err := host.Info()
if err != nil {
return 0, err
@ -39,7 +39,7 @@ func (s *SystemInfo) Uptime() (uptime uint64, err error) {
}
// collectPlatformInfo fetches and caches system platform information.
func (s *SystemInfo) collectPlatformInfo() error {
func (s *systemInfo) collectPlatformInfo() error {
var err error
if s.platform == "" {
s.platform, s.family, s.osVersion, err = host.PlatformInformation()
@ -51,7 +51,7 @@ func (s *SystemInfo) collectPlatformInfo() error {
}
// Platform returns the system platform.
func (s *SystemInfo) Platform() (string, error) {
func (s *systemInfo) Platform() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
@ -60,7 +60,7 @@ func (s *SystemInfo) Platform() (string, error) {
}
// Family returns the system family.
func (s *SystemInfo) Family() (string, error) {
func (s *systemInfo) Family() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
@ -69,7 +69,7 @@ func (s *SystemInfo) Family() (string, error) {
}
// OSVersion returns the OS Version.
func (s *SystemInfo) OSVersion() (string, error) {
func (s *systemInfo) OSVersion() (string, error) {
err := s.collectPlatformInfo()
if err != nil {
return "", err
@ -78,7 +78,7 @@ func (s *SystemInfo) OSVersion() (string, error) {
}
// MemFree returns the amount of free memory in bytes.
func (s *SystemInfo) MemFree() (uint64, error) {
func (s *systemInfo) MemFree() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
@ -87,7 +87,7 @@ func (s *SystemInfo) MemFree() (uint64, error) {
}
// MemTotal returns the amount of total memory in bytes.
func (s *SystemInfo) MemTotal() (uint64, error) {
func (s *systemInfo) MemTotal() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
@ -96,7 +96,7 @@ func (s *SystemInfo) MemTotal() (uint64, error) {
}
// MemUsed returns the amount of used memory in bytes.
func (s *SystemInfo) MemUsed() (uint64, error) {
func (s *systemInfo) MemUsed() (uint64, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
return 0, err
@ -105,11 +105,11 @@ func (s *SystemInfo) MemUsed() (uint64, error) {
}
// KernelVersion returns the kernel version as a string.
func (s *SystemInfo) KernelVersion() (string, error) {
func (s *systemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *SystemInfo {
return &SystemInfo{}
func NewSystemInfo() *systemInfo {
return &systemInfo{}
}

76
gossip/gossip.go
View file

@ -26,19 +26,17 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
// Ensure GossipMemberSet implements interfaces.
var _ memberlist.Delegate = &GossipMemberSet{}
var _ memberlist.Delegate = &gossipMemberSet{}
// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
type GossipMemberSet struct {
// gossipMemberSet represents a gossip implementation of MemberSet using memberlist.
type gossipMemberSet struct {
mu sync.RWMutex
memberlist *memberlist.Memberlist
@ -56,7 +54,7 @@ type GossipMemberSet struct {
}
// Open implements the MemberSet interface to start network activity.
func (g *GossipMemberSet) Open() (err error) {
func (g *gossipMemberSet) Open() (err error) {
g.mu.Lock()
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
@ -96,7 +94,7 @@ func (g *GossipMemberSet) Open() (err error) {
}
// joinWithRetry wraps the standard memberlist Join function in a retry.
func (g *GossipMemberSet) joinWithRetry(hosts []string) error {
func (g *gossipMemberSet) joinWithRetry(hosts []string) error {
err := retry(60, 2*time.Second, func() error {
_, err := g.memberlist.Join(hosts)
return err
@ -127,29 +125,29 @@ type gossipConfig struct {
memberlistConfig *memberlist.Config
}
// GossipMemberSetOption describes a functional option for GossipMemberSet.
type GossipMemberSetOption func(*GossipMemberSet) error
// gossipMemberSetOption describes a functional option for GossipMemberSet.
type gossipMemberSetOption func(*gossipMemberSet) error
// WithTransport is a functional option for providing a transport to NewGossipMemberSet.
func WithTransport(transport *Transport) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
func WithTransport(transport *Transport) gossipMemberSetOption {
return func(g *gossipMemberSet) error {
g.transport = transport
return nil
}
}
// WithLogger is a functional option for providing a logger to NewGossipMemberSet.
func WithLogger(logger *log.Logger) GossipMemberSetOption {
return func(g *GossipMemberSet) error {
func WithLogger(logger *log.Logger) gossipMemberSetOption {
return func(g *gossipMemberSet) error {
g.logger = logger
return nil
}
}
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
host := api.Node().URI.Host()
g := &GossipMemberSet{
func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetOption) (*gossipMemberSet, error) {
host := api.Node().URI.Host
g := &gossipMemberSet{
papi: api,
Logger: pilosa.NopLogger,
}
@ -178,7 +176,7 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
g.transport = transport
}
port := g.transport.Net.GetAutoBindPort()
port := g.transport.net.GetAutoBindPort()
var gossipKey []byte
var err error
@ -191,12 +189,12 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = g.transport.Net
conf.Transport = g.transport.net
conf.Name = api.Node().ID
conf.BindAddr = api.Node().URI.Host()
conf.BindAddr = api.Node().URI.Host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = hostToIP(api.Node().URI.Host())
conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
@ -221,8 +219,8 @@ func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetO
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.papi.Node()))
func (g *gossipMemberSet) NodeMeta(limit int) []byte {
buf, err := g.papi.Serializer.Marshal(g.papi.Node())
if err != nil {
g.Logger.Printf("marshal message error: %s", err)
return []byte{}
@ -232,7 +230,7 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte {
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipMemberSet) NotifyMsg(b []byte) {
func (g *gossipMemberSet) NotifyMsg(b []byte) {
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b))
if err != nil {
g.Logger.Printf("cluster message error: %s", err)
@ -241,21 +239,21 @@ func (g *GossipMemberSet) NotifyMsg(b []byte) {
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
func (g *gossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb := &internal.NodeStatus{
Node: pilosa.EncodeNode(g.papi.Node()),
MaxShards: &internal.MaxShards{Standard: g.papi.MaxShards(context.Background())},
Schema: &internal.Schema{Indexes: pilosa.EncodeIndexes(g.papi.Schema(context.Background()))},
func (g *gossipMemberSet) LocalState(join bool) []byte {
m := &pilosa.NodeStatus{
Node: g.papi.Node(),
MaxShards: g.papi.MaxShards(context.Background()),
Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
}
// Marshal nodestate data to bytes.
buf, err := pilosa.MarshalMessage(pb)
buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer)
if err != nil {
g.Logger.Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
@ -265,7 +263,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte {
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side's LocalState.
func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
func (g *gossipMemberSet) MergeRemoteState(buf []byte, join bool) {
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf))
if err != nil {
g.Logger.Printf("merge state error: %s", err)
@ -323,16 +321,16 @@ func (g *gossipEventReceiver) listen() {
}
// Get the node from the event.Node meta data.
var n internal.Node
if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta data")
var n pilosa.Node
if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta into node")
}
ne := &internal.NodeEventMessage{
Event: uint32(nodeEventType),
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: &n,
}
buf, err := pilosa.MarshalMessage(ne)
buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer)
if err != nil {
panic(err)
}
@ -345,7 +343,7 @@ func (g *gossipEventReceiver) listen() {
// Transport is a gossip transport for binding to a port.
type Transport struct {
//memberlist.Transport
Net *memberlist.NetTransport
net *memberlist.NetTransport
URI *pilosa.URI
}
@ -372,7 +370,7 @@ func NewTransport(host string, port int, logger *log.Logger) (*Transport, error)
}
return &Transport{
Net: net,
net: net,
URI: uri,
}, nil
}

37
handler.go
View file

@ -75,3 +75,40 @@ func (n nopHandler) Close() error {
}
var NopHandler Handler = nopHandler{}
type ImportValueRequest struct {
Index string
Field string
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
}
type ImportRequest struct {
Index string
Field string
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
}
type ImportResponse struct {
Err string
}
type BlockDataRequest struct {
Index string
Field string
View string
Shard uint64
Block uint64
}
type BlockDataResponse struct {
RowIDs []uint64
ColumnIDs []uint64
}

50
holder.go
View file

@ -27,7 +27,6 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -217,7 +216,7 @@ func (h *Holder) Schema() []*IndexInfo {
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.views() {
fi.Views = append(fi.Views, &viewInfo{Name: view.name})
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Fields = append(di.Fields, fi)
@ -229,8 +228,24 @@ func (h *Holder) Schema() []*IndexInfo {
return a
}
// limitedSchema returns schema information for all indexes and fields.
func (h *Holder) limitedSchema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
di.Fields = append(di.Fields, fi)
}
sort.Sort(fieldInfoSlice(di.Fields))
a = append(a, di)
}
sort.Sort(indexInfoSlice(a))
return a
}
// applySchema applies an internal Schema to Holder.
func (h *Holder) applySchema(schema *internal.Schema) error {
func (h *Holder) applySchema(schema *Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
opt := IndexOptions{}
@ -240,14 +255,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
}
// Create fields that don't exist.
for _, f := range index.Fields {
opt := decodeFieldOptions(f.Meta)
field, err := idx.createFieldIfNotExists(f.Name, *opt)
field, err := idx.createFieldIfNotExists(f.Name, f.Options)
if err != nil {
return errors.Wrap(err, "creating field")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := field.createViewIfNotExists(v)
_, err := field.createViewIfNotExists(v.Name)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -257,20 +271,6 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
return nil
}
// encodeMaxShards creates and internal representation of max shards.
func (h *Holder) encodeMaxShards() *internal.MaxShards {
return &internal.MaxShards{
Standard: h.maxShards(),
}
}
// encodeSchema creates an internal representation of schema.
func (h *Holder) encodeSchema() *internal.Schema {
return &internal.Schema{
Indexes: EncodeIndexes(h.Indexes()),
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -304,7 +304,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
// Ensure index doesn't already exist.
if h.indexes[name] != nil {
return nil, NewConflictError(ErrIndexExists)
return nil, newConflictError(ErrIndexExists)
}
return h.createIndex(name, opt)
}
@ -374,7 +374,7 @@ func (h *Holder) DeleteIndex(name string) error {
// Confirm index exists.
index := h.index(name)
if index == nil {
return NewNotFoundError(ErrIndexNotFound)
return newNotFoundError(ErrIndexNotFound)
}
// Close index.
@ -456,13 +456,13 @@ func (h *Holder) flushCaches() {
}
}
// RecalculateCaches recalculates caches on every index in the holder. This is
// recalculateCaches recalculates caches on every index in the holder. This is
// probably not practical to call in real-world workloads, but makes writing
// integration tests much eaiser, since one doesn't have to wait 10 seconds
// after setting bits to get expected response.
func (h *Holder) RecalculateCaches() {
func (h *Holder) recalculateCaches() {
for _, index := range h.Indexes() {
index.RecalculateCaches()
index.recalculateCaches()
}
}

103
http/client.go
View file

@ -27,25 +27,18 @@ import (
"sort"
"strconv"
"crypto/tls"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pkg/errors"
)
// ClientOptions represents the configuration for a InternalHTTPClient
type ClientOptions struct {
TLS *tls.Config
}
// InternalClient represents a client to the Pilosa cluster.
type InternalClient struct {
defaultURI *pilosa.URI
serializer pilosa.Serializer
// The client to use for HTTP communication.
HTTPClient *http.Client
httpClient *http.Client
}
// NewInternalClient returns a new instance of InternalClient to connect to host.
@ -66,13 +59,11 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
return &InternalClient{
defaultURI: defaultURI,
HTTPClient: remoteClient,
serializer: proto.Serializer{},
httpClient: remoteClient,
}
}
// Host returns the host the client was initialized with.
func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI }
// MaxShardByIndex returns the number of shards on a server by index.
func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxShardByIndex(ctx)
@ -93,7 +84,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -124,7 +115,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -161,7 +152,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
@ -200,7 +191,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -217,22 +208,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
}
// Query executes query against the index.
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
if index == "" {
return nil, pilosa.ErrIndexRequired
} else if queryRequest.Query == "" {
return nil, pilosa.ErrQueryRequired
}
// Encode request object.
buf, err := proto.Marshal(queryRequest)
buf, err := c.serializer.Marshal(queryRequest)
if err != nil {
return nil, errors.Wrap(err, "marshaling")
return nil, errors.Wrap(err, "marshaling queryRequest")
}
// Create HTTP request.
@ -248,7 +238,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -262,11 +252,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
return nil, errors.New(string(body))
}
qresp := &internal.QueryResponse{}
if err := proto.Unmarshal(body, qresp); err != nil {
qresp := &pilosa.QueryResponse{}
if err := c.serializer.Unmarshal(body, qresp); err != nil {
return nil, fmt.Errorf("unmarshal response: %s", err)
} else if s := qresp.Err; s != "" {
return nil, errors.New(s)
} else if qresp.Err != nil {
return nil, qresp.Err
}
return qresp, nil
@ -280,7 +270,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard
return pilosa.ErrFieldRequired
}
buf, err := marshalImportPayload(index, field, shard, bits)
buf, err := c.marshalImportPayload(index, field, shard, bits)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -309,7 +299,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum
return pilosa.ErrFieldRequired
}
buf, err := marshalImportPayloadK(index, field, columns)
buf, err := c.marshalImportPayloadK(index, field, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -343,14 +333,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel
}
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
Index: index,
Field: field,
Shard: shard,
@ -365,14 +355,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit)
}
// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowKeys := Bits(bits).RowKeys()
columnKeys := Bits(bits).ColumnKeys()
timestamps := Bits(bits).Timestamps()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
Index: index,
Field: field,
RowKeys: rowKeys,
@ -400,7 +390,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
@ -414,8 +404,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
return errors.New(string(body))
}
var isresp internal.ImportResponse
if err := proto.Unmarshal(body, &isresp); err != nil {
var isresp pilosa.ImportResponse
if err := c.serializer.Unmarshal(body, &isresp); err != nil {
return fmt.Errorf("unmarshal import response: %s", err)
} else if s := isresp.Err; s != "" {
return errors.New(s)
@ -432,7 +422,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
return pilosa.ErrFieldRequired
}
buf, err := marshalImportValuePayload(index, field, shard, vals)
buf, err := c.marshalImportValuePayload(index, field, shard, vals)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -454,13 +444,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
}
// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportValueRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{
Index: index,
Field: field,
Shard: shard,
@ -522,7 +512,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
@ -565,7 +555,7 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -609,7 +599,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
@ -655,7 +645,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -683,7 +673,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
if uri == nil {
panic("need to pass a URI to BlockData")
}
buf, err := proto.Marshal(&internal.BlockDataRequest{
buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{
Index: index,
Field: field,
Shard: shard,
@ -703,7 +693,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
req.Header.Set("Accept", "application/protobuf")
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, nil, errors.Wrap(err, "executing request")
}
@ -719,10 +709,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
}
// Decode response object.
var rsp internal.BlockDataResponse
var rsp pilosa.BlockDataResponse
if body, err := ioutil.ReadAll(resp.Body); err != nil {
return nil, nil, errors.Wrap(err, "reading")
} else if err := proto.Unmarshal(body, &rsp); err != nil {
} else if err := c.serializer.Unmarshal(body, &rsp); err != nil {
return nil, nil, errors.Wrap(err, "unmarshalling")
}
return rsp.RowIDs, rsp.ColumnIDs, nil
@ -751,7 +741,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -795,7 +785,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
@ -819,12 +809,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
}
// SendMessage posts a message synchronously.
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
u := uriPathToURL(uri, "/internal/cluster/message")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
if err != nil {
@ -835,7 +820,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb pr
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return fmt.Errorf("executing http request: %v", err)
}
@ -998,7 +983,7 @@ func pos(rowID, columnID uint64) uint64 {
func uriPathToURL(uri *pilosa.URI, path string) url.URL {
return url.URL{
Scheme: uri.Scheme(),
Scheme: uri.Scheme,
Host: uri.HostPort(),
Path: path,
}
@ -1006,7 +991,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL {
func nodePathToURL(node *pilosa.Node, path string) url.URL {
return url.URL{
Scheme: node.URI.Scheme(),
Scheme: node.URI.Scheme,
Host: node.URI.HostPort(),
Path: path,
}

9
http/client_test.go
View file

@ -24,7 +24,6 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) {
client[2] = MustNewClient(c[2].URL(), defaultClient)
topN := 4
queryRequest := &internal.QueryRequest{
queryRequest := &pilosa.QueryRequest{
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) {
}
// Test must return exactly N results.
if len(result.Results[0].Pairs) != topN {
if len(result.Results[0].([]pilosa.Pair)) != topN {
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
}
p := []*internal.Pair{
p := []pilosa.Pair{
{ID: 100, Count: 12},
{ID: 22, Count: 10},
{ID: 98, Count: 8},
{ID: 99, Count: 7}}
// Valdidate the Top 4 result counts.
if !reflect.DeepEqual(result.Results[0].Pairs, p) {
if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}

2
http/error.go
View file

@ -17,7 +17,7 @@ package http
// Error defines a standard application error.
type Error struct {
// Machine-readable error code.
Code string `json:"code,omitempty"`
code string `json:"code,omitempty"`
// Human-readable message.
Message string `json:"message"`

234
http/handler.go
View file

@ -33,11 +33,9 @@ import (
"strings"
"time"
"github.com/gogo/protobuf/proto"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -46,14 +44,14 @@ import (
type Handler struct {
Handler http.Handler
Logger pilosa.Logger
logger pilosa.Logger
// Keeps the query argument validators for each handler
validators map[string]*queryValidationSpec
API *pilosa.API
api *pilosa.API
AllowedOrigins []string
allowedOrigins []string
ln net.Listener
@ -77,10 +75,10 @@ type errorResponse struct {
Error string `json:"error"`
}
// HandlerOption is a functional option type for pilosa.Handler
type HandlerOption func(s *Handler) error
// handlerOption is a functional option type for pilosa.Handler
type handlerOption func(s *Handler) error
func OptHandlerAllowedOrigins(origins []string) HandlerOption {
func OptHandlerAllowedOrigins(origins []string) handlerOption {
return func(h *Handler) error {
h.Handler = handlers.CORS(
handlers.AllowedOrigins(origins),
@ -90,21 +88,21 @@ func OptHandlerAllowedOrigins(origins []string) HandlerOption {
}
}
func OptHandlerAPI(api *pilosa.API) HandlerOption {
func OptHandlerAPI(api *pilosa.API) handlerOption {
return func(h *Handler) error {
h.API = api
h.api = api
return nil
}
}
func OptHandlerLogger(logger pilosa.Logger) HandlerOption {
func OptHandlerLogger(logger pilosa.Logger) handlerOption {
return func(h *Handler) error {
h.Logger = logger
h.logger = logger
return nil
}
}
func OptHandlerListener(ln net.Listener) HandlerOption {
func OptHandlerListener(ln net.Listener) handlerOption {
return func(h *Handler) error {
h.ln = ln
return nil
@ -112,11 +110,11 @@ func OptHandlerListener(ln net.Listener) HandlerOption {
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler(opts ...HandlerOption) (*Handler, error) {
func NewHandler(opts ...handlerOption) (*Handler, error) {
handler := &Handler{
Logger: pilosa.NopLogger,
logger: pilosa.NopLogger,
}
handler.Handler = NewRouter(handler)
handler.Handler = newRouter(handler)
handler.populateValidators()
for _, opt := range opts {
@ -126,7 +124,7 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
}
}
if handler.API == nil {
if handler.api == nil {
return nil, errors.New("must pass OptHandlerAPI")
}
@ -142,7 +140,7 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
func (h *Handler) Serve() error {
err := h.server.Serve(h.ln)
if err != nil && err.Error() != "http: Server closed" {
h.Logger.Printf("HTTP handler terminated with error: %s\n", err)
h.logger.Printf("HTTP handler terminated with error: %s\n", err)
return errors.Wrap(err, "serve http")
}
return nil
@ -185,8 +183,8 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
})
}
// NewRouter creates a new mux http router.
func NewRouter(handler *Handler) *mux.Router {
// newRouter creates a new mux http router.
func newRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
router.HandleFunc("/", handler.handleHome).Methods("GET")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
@ -242,7 +240,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
stack := debug.Stack()
msg := "PANIC: %s\n%s"
h.Logger.Printf(msg, err, stack)
h.logger.Printf(msg, err, stack)
fmt.Fprintf(w, msg, err, stack)
}
}()
@ -254,9 +252,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Calculate per request StatsD metrics when the handler is fully configured.
statsTags := make([]string, 0, 3)
longQueryTime := h.API.LongQueryTime()
longQueryTime := h.api.LongQueryTime()
if longQueryTime > 0 && dif > longQueryTime {
h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif)
h.logger.Printf("%s %s %v", r.Method, r.URL.String(), dif)
statsTags = append(statsTags, "slow_query")
}
@ -269,7 +267,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// useragent tag identifies internal/external endpoints
statsTags = append(statsTags, "useragent:"+r.UserAgent())
stats := h.API.StatsWithTags(statsTags)
stats := h.api.StatsWithTags(statsTags)
if stats != nil {
stats.Histogram("http."+endpointName, float64(dif), 0.1)
}
@ -357,9 +355,9 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
return
}
schema := h.API.Schema(r.Context())
schema := h.api.Schema(r.Context())
if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil {
h.Logger.Printf("write schema response error: %s", err)
h.logger.Printf("write schema response error: %s", err)
}
}
@ -370,12 +368,12 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
return
}
status := getStatusResponse{
State: h.API.State(),
Nodes: h.API.Hosts(r.Context()),
LocalID: h.API.Node().ID,
State: h.api.State(),
Nodes: h.api.Hosts(r.Context()),
LocalID: h.api.Node().ID,
}
if err := json.NewEncoder(w).Encode(status); err != nil {
h.Logger.Printf("write status response error: %s", err)
h.logger.Printf("write status response error: %s", err)
}
}
@ -384,9 +382,9 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
info := h.API.Info()
info := h.api.Info()
if err := json.NewEncoder(w).Encode(info); err != nil {
h.Logger.Printf("write info response error: %s", err)
h.logger.Printf("write info response error: %s", err)
}
}
@ -412,26 +410,33 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// TODO: Remove
req.Index = mux.Vars(r)["index"]
resp, err := h.API.Query(r.Context(), req)
resp, err := h.api.Query(r.Context(), req)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
switch errors.Cause(resp.Err) {
case pilosa.ErrTooManyWrites:
w.WriteHeader(http.StatusRequestEntityTooLarge)
default:
w.WriteHeader(http.StatusBadRequest)
}
h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err})
return
}
// Set appropriate status code, if there is an error.
// Set appropriate status code, if there is an error. It doesn't appear that
// resp.Err could ever be set in API.Query, so this code block is probably
// doing nothing right now.
if resp.Err != nil {
switch resp.Err {
switch errors.Cause(resp.Err) {
case pilosa.ErrTooManyWrites:
w.WriteHeader(http.StatusRequestEntityTooLarge)
default:
w.WriteHeader(http.StatusInternalServerError)
w.WriteHeader(http.StatusBadRequest)
}
}
// Write response back to client.
if err := h.writeQueryResponse(w, r, &resp); err != nil {
h.Logger.Printf("write query response error: %s", err)
h.logger.Printf("write query response error: %s", err)
}
}
@ -442,9 +447,9 @@ func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) {
return
}
if err := json.NewEncoder(w).Encode(getShardsMaxResponse{
Standard: h.API.MaxShards(r.Context()),
Standard: h.api.MaxShards(r.Context()),
}); err != nil {
h.Logger.Printf("write shards-max response error: %s", err)
h.logger.Printf("write shards-max response error: %s", err)
}
}
@ -464,10 +469,10 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
return
}
indexName := mux.Vars(r)["index"]
for _, idx := range h.API.Schema(r.Context()) {
if idx.Name() == indexName {
for _, idx := range h.api.Schema(r.Context()) {
if idx.Name == indexName {
if err := json.NewEncoder(w).Encode(idx); err != nil {
h.Logger.Printf("write response error: %s", err)
h.logger.Printf("write response error: %s", err)
}
return
}
@ -558,7 +563,7 @@ func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
resp := successResponse{}
err := h.API.DeleteIndex(r.Context(), indexName)
err := h.api.DeleteIndex(r.Context(), indexName)
resp.write(w, err)
}
@ -579,7 +584,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
return
}
_, err = h.API.CreateIndex(r.Context(), indexName, req.Options)
_, err = h.api.CreateIndex(r.Context(), indexName, req.Options)
resp.write(w, err)
}
@ -599,7 +604,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
return
}
attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks)
attrs, err := h.api.IndexAttrDiff(r.Context(), indexName, req.Blocks)
if err != nil {
if errors.Cause(err) == pilosa.ErrIndexNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
@ -613,7 +618,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
@ -668,7 +673,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
}
}
_, err = h.API.CreateField(r.Context(), indexName, fieldName, fos...)
_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
resp.write(w, err)
}
@ -676,7 +681,7 @@ type postFieldRequest struct {
Options fieldOptions `json:"options"`
}
// fieldOptions tracks pilosa.fieldOptions. It is made up of pointers to values,
// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values,
// and used for input validation.
type fieldOptions struct {
Type string `json:"type,omitempty"`
@ -755,7 +760,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) {
fieldName := mux.Vars(r)["field"]
resp := successResponse{}
err := h.API.DeleteField(r.Context(), indexName, fieldName)
err := h.api.DeleteField(r.Context(), indexName, fieldName)
resp.write(w, err)
}
@ -775,7 +780,7 @@ func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request
return
}
attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks)
attrs, err := h.api.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks)
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrFragmentNotFound:
@ -790,7 +795,7 @@ func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
@ -820,13 +825,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques
return nil, errors.Wrap(err, "reading")
}
// Unmarshal into object.
var req internal.QueryRequest
if err := proto.Unmarshal(body, &req); err != nil {
return nil, errors.Wrap(err, "unmarshalling")
qreq := &pilosa.QueryRequest{}
err = h.api.Serializer.Unmarshal(body, qreq)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling query request")
}
return decodeQueryRequest(&req), nil
return qreq, nil
}
// readURLQueryRequest parses query parameters from URL parameters from r.
@ -865,7 +869,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res
// writeProtobufQueryResponse writes the response from the executor to w as protobuf.
func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error {
if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil {
if buf, err := h.api.Serializer.Marshal(resp); err != nil {
return errors.Wrap(err, "marshalling")
} else if _, err := w.Write(buf); err != nil {
return errors.Wrap(err, "writing")
@ -893,7 +897,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Get index and field type to determine how to handle the
// import data.
field, err := h.API.Field(r.Context(), indexName, fieldName)
field, err := h.api.Field(r.Context(), indexName, fieldName)
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrIndexNotFound:
@ -917,13 +921,13 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
if field.Type() == pilosa.FieldTypeInt {
// Field type: Int
// Marshal into request object.
var req internal.ImportValueRequest
if err := proto.Unmarshal(body, &req); err != nil {
req := &pilosa.ImportValueRequest{}
if err := h.api.Serializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.API.ImportValue(r.Context(), req); err != nil {
if err := h.api.ImportValue(r.Context(), req); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -935,13 +939,13 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
} else {
// Field type: Set, Time
// Marshal into request object.
var req internal.ImportRequest
if err := proto.Unmarshal(body, &req); err != nil {
req := &pilosa.ImportRequest{}
if err := h.api.Serializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.API.Import(r.Context(), req); err != nil {
if err := h.api.Import(r.Context(), req); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -953,7 +957,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Marshal response object.
buf, e := proto.Marshal(&internal.ImportResponse{Err: ""})
buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
if e != nil {
http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError)
return
@ -984,7 +988,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
return
}
if err = h.API.ExportCSV(r.Context(), index, field, shard, w); err != nil {
if err = h.api.ExportCSV(r.Context(), index, field, shard, w); err != nil {
switch errors.Cause(err) {
case pilosa.ErrFragmentNotFound:
break
@ -1014,7 +1018,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
// Retrieve fragment owner nodes.
nodes, err := h.API.ShardNodes(r.Context(), index, shard)
nodes, err := h.api.ShardNodes(r.Context(), index, shard)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -1022,13 +1026,13 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
// Write to response.
if err := json.NewEncoder(w).Encode(nodes); err != nil {
h.Logger.Printf("json write error: %s", err)
h.logger.Printf("json write error: %s", err)
}
}
// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
buf, err := h.API.FragmentBlockData(r.Context(), r.Body)
buf, err := h.api.FragmentBlockData(r.Context(), r.Body)
if err != nil {
if _, ok := err.(pilosa.BadRequestError); ok {
http.Error(w, err.Error(), http.StatusBadRequest)
@ -1060,7 +1064,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
return
}
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard)
blocks, err := h.api.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard)
if err != nil {
if errors.Cause(err) == pilosa.ErrFragmentNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
@ -1074,7 +1078,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{
Blocks: blocks,
}); err != nil {
h.Logger.Printf("block response encoding error: %s", err)
h.logger.Printf("block response encoding error: %s", err)
}
}
@ -1091,73 +1095,23 @@ func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
err := json.NewEncoder(w).Encode(struct {
Version string `json:"version"`
}{
Version: h.API.Version(),
Version: h.api.Version(),
})
if err != nil {
h.Logger.Printf("write version response error: %s", err)
h.logger.Printf("write version response error: %s", err)
}
}
// QueryResult types.
const (
QueryResultTypeNil uint32 = iota
queryResultTypeNil uint32 = iota
QueryResultTypeRow
QueryResultTypePairs
QueryResultTypeValCount
queryResultTypeValCount
QueryResultTypeUint64
QueryResultTypeBool
queryResultTypeBool
)
func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest {
req := &pilosa.QueryRequest{
Query: pb.Query,
Shards: pb.Shards,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,
ExcludeColumns: pb.ExcludeColumns,
}
return req
}
func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets),
}
for i := range resp.Results {
pb.Results[i] = &internal.QueryResult{}
switch result := resp.Results[i].(type) {
case *pilosa.Row:
pb.Results[i].Type = QueryResultTypeRow
pb.Results[i].Row = pilosa.EncodeRow(result)
case []pilosa.Pair:
pb.Results[i].Type = QueryResultTypePairs
pb.Results[i].Pairs = pilosa.EncodePairs(result)
case pilosa.ValCount:
pb.Results[i].Type = QueryResultTypeValCount
pb.Results[i].ValCount = pilosa.EncodeValCount(result)
case uint64:
pb.Results[i].Type = QueryResultTypeUint64
pb.Results[i].N = result
case bool:
pb.Results[i].Type = QueryResultTypeBool
pb.Results[i].Changed = result
case nil:
pb.Results[i].Type = QueryResultTypeNil
}
}
if resp.Err != nil {
pb.Err = resp.Err.Error()
}
return pb
}
// parseUint64Slice returns a slice of uint64s from a comma-delimited string.
func parseUint64Slice(s string) ([]uint64, error) {
var a []uint64
@ -1198,7 +1152,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r
return
}
oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID)
oldNode, newNode, err := h.api.SetCoordinator(r.Context(), req.ID)
if err != nil {
if errors.Cause(err) == pilosa.ErrNodeIDNotExists {
http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound)
@ -1212,7 +1166,7 @@ func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r
Old: oldNode,
New: newNode,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
@ -1239,7 +1193,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht
return
}
removeNode, err := h.API.RemoveNode(req.ID)
removeNode, err := h.api.RemoveNode(req.ID)
if err != nil {
if errors.Cause(err) == pilosa.ErrNodeIDNotExists {
http.Error(w, "removing node: "+err.Error(), http.StatusNotFound)
@ -1253,7 +1207,7 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht
if err := json.NewEncoder(w).Encode(removeNodeResponse{
Remove: removeNode,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
@ -1271,7 +1225,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
err := h.API.ResizeAbort()
err := h.api.ResizeAbort()
var msg string
if err != nil {
switch errors.Cause(err) {
@ -1289,7 +1243,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re
if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{
Info: msg,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
@ -1298,7 +1252,7 @@ type clusterResizeAbortResponse struct {
}
func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) {
err := h.API.RecalculateCaches(r.Context())
err := h.api.RecalculateCaches(r.Context())
if err != nil {
http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError)
return
@ -1318,21 +1272,17 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
return
}
err := h.API.ClusterMessage(r.Context(), r.Body)
err := h.api.ClusterMessage(r.Context(), r.Body)
if err != nil {
// TODO this was the previous behavior, but perhaps not everything is a bad request
http.Error(w, err.Error(), http.StatusBadRequest)
}
if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
h.logger.Printf("response encoding error: %s", err)
}
}
func (h *Handler) GetAPI() *pilosa.API {
return h.API
}
type defaultClusterMessageResponse struct{}
func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
@ -1341,7 +1291,7 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
pipeR, pipeW := io.Pipe()
err := h.API.GetTranslateData(r.Context(), pipeW, offset)
err := h.api.GetTranslateData(r.Context(), pipeW, offset)
if err != nil {
if errors.Cause(err) == pilosa.ErrNotImplemented {

20
http/translator.go
View file

@ -14,41 +14,41 @@ import (
)
// Ensure implementation implements inteface.
var _ pilosa.TranslateStore = (*TranslateStore)(nil)
var _ pilosa.TranslateStore = (*translateStore)(nil)
// TranslateStore represents an implementation of TranslateStore that
// translateStore represents an implementation of translateStore that
// communicates over HTTP. This is used with the TranslateHandler.
type TranslateStore struct {
type translateStore struct {
URL string
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(rawurl string) *TranslateStore {
return &TranslateStore{URL: rawurl}
func NewTranslateStore(rawurl string) *translateStore {
return &translateStore{URL: rawurl}
}
// TranslateColumnsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateColumnToString is not currently implemented.
func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) {
func (s *translateStore) TranslateColumnToString(index string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// TranslateRowsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateRowToString is not currently implemented.
func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) {
func (s *translateStore) TranslateRowToString(index, frame string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// Reader returns a reader that can stream data from a remote store.
func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
func (s *translateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
// Generate remote URL.
u, err := url.Parse(s.URL)
if err != nil {

78
index.go
View file

@ -15,7 +15,6 @@
package pilosa
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
@ -76,22 +75,6 @@ func NewIndex(path, name string) (*Index, error) {
}, nil
}
func (i *Index) MarshalJSON() ([]byte, error) {
fields := make([]*Field, 0, len(i.fields))
for _, f := range i.fields {
fields = append(fields, f)
}
thing := struct {
Name string
Fields []*Field
}{
Name: i.name,
Fields: fields,
}
return json.Marshal(thing)
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
@ -156,7 +139,7 @@ func (i *Index) openFields() error {
continue
}
fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
@ -237,7 +220,7 @@ func (i *Index) maxShard() uint64 {
max := i.remoteMaxShard
for _, f := range i.fields {
if shard := f.MaxShard(); shard > max {
if shard := f.maxShard(); shard > max {
max = shard
}
}
@ -253,8 +236,8 @@ func (i *Index) setRemoteMaxShard(newmax uint64) {
i.remoteMaxShard = newmax
}
// FieldPath returns the path to a field in the index.
func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) }
// fieldPath returns the path to a field in the index.
func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) }
// Field returns a field in the index by name.
func (i *Index) Field(name string) *Field {
@ -279,10 +262,10 @@ func (i *Index) Fields() []*Field {
return a
}
// RecalculateCaches recalculates caches on every field in the index.
func (i *Index) RecalculateCaches() {
// recalculateCaches recalculates caches on every field in the index.
func (i *Index) recalculateCaches() {
for _, field := range i.Fields() {
field.RecalculateCaches()
field.recalculateCaches()
}
}
@ -293,11 +276,11 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
// Ensure field doesn't already exist.
if i.fields[name] != nil {
return nil, NewConflictError(ErrFieldExists)
return nil, newConflictError(ErrFieldExists)
}
// Apply functional options.
fo := fieldOptions{}
fo := FieldOptions{}
for _, opt := range opts {
err := opt(&fo)
if err != nil {
@ -319,7 +302,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e
}
// Apply functional option.
fo := fieldOptions{}
fo := FieldOptions{}
err := opts(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
@ -328,7 +311,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e
return i.createField(name, fo)
}
func (i *Index) createFieldIfNotExists(name string, opt fieldOptions) (*Field, error) {
func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
@ -340,7 +323,7 @@ func (i *Index) createFieldIfNotExists(name string, opt fieldOptions) (*Field, e
return i.createField(name, opt)
}
func (i *Index) createField(name string, opt fieldOptions) (*Field, error) {
func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
if name == "" {
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
@ -348,7 +331,7 @@ func (i *Index) createField(name string, opt fieldOptions) (*Field, error) {
}
// Initialize field.
f, err := i.newField(i.FieldPath(name), name)
f, err := i.newField(i.fieldPath(name), name)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
@ -380,7 +363,7 @@ func (i *Index) newField(path, name string) (*Field, error) {
if err != nil {
return nil, err
}
f.Logger = i.logger
f.logger = i.logger
f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
@ -395,7 +378,7 @@ func (i *Index) DeleteField(name string) error {
// Confirm field exists.
f := i.field(name)
if f == nil {
return NewNotFoundError(ErrFieldNotFound)
return newNotFoundError(ErrFieldNotFound)
}
// Close field.
@ -404,7 +387,7 @@ func (i *Index) DeleteField(name string) error {
}
// Delete field directory.
if err := os.RemoveAll(i.FieldPath(name)); err != nil {
if err := os.RemoveAll(i.fieldPath(name)); err != nil {
return errors.Wrap(err, "removing directory")
}
@ -422,8 +405,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
Fields []*FieldInfo `json:"fields"`
Name string `json:"name"`
options IndexOptions `json:"options"`
Fields []*FieldInfo `json:"fields"`
}
type indexInfoSlice []*IndexInfo
@ -432,35 +416,11 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexInfoSlice) Len() int { return len(p) }
func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// EncodeIndexes converts a into its internal representation.
func EncodeIndexes(a []*Index) []*internal.Index {
other := make([]*internal.Index, len(a))
for i := range a {
other[i] = encodeIndex(a[i])
}
return other
}
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Fields: encodeFields(d.Fields()),
}
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
Keys bool `json:"keys"`
}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
Keys: i.Keys,
}
}
// hasTime returns true if a contains a non-nil time.
func hasTime(a []*time.Time) bool {
for _, t := range a {

20
inmem/translator.go
View file

@ -9,10 +9,10 @@ import (
)
// Ensure type implements interface.
var _ pilosa.TranslateStore = &TranslateStore{}
var _ pilosa.TranslateStore = &translateStore{}
// TranslateStore is an in-memory storage engine for translating string-to-uint64 values.
type TranslateStore struct {
// translateStore is an in-memory storage engine for translating string-to-uint64 values.
type translateStore struct {
mu sync.RWMutex
cols map[string]*translateIndex
@ -20,21 +20,21 @@ type TranslateStore struct {
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore() *TranslateStore {
return &TranslateStore{
func NewTranslateStore() *translateStore {
return &translateStore{
cols: make(map[string]*translateIndex),
rows: make(map[frameKey]*translateIndex),
}
}
// Reader returns an error because it is not supported by the inmem store.
func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
func (s *translateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
return nil, pilosa.ErrReplicationNotSupported
}
// TranslateColumnsToUint64 converts value to a uint64 id.
// If value does not have an associated id then one is created.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
ret := make([]uint64, len(values))
// Read value under read lock.
@ -103,7 +103,7 @@ func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string)
// TranslateColumnToString converts a uint64 id to its associated string value.
// If the id is not associated with a string value then a blank string is returned.
func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) {
func (s *translateStore) TranslateColumnToString(index string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.cols[index]; idx != nil {
if ret, ok := idx.reverse[value]; ok {
@ -115,7 +115,7 @@ func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (st
return "", nil
}
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
key := frameKey{index, frame}
ret := make([]uint64, len(values))
@ -184,7 +184,7 @@ func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []str
return ret, nil
}
func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
func (s *translateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.rows[frameKey{index, frame}]; idx != nil {
if ret, ok := idx.reverse[value]; ok {

28
logger.go
View file

@ -43,46 +43,46 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {}
// Debugf is a no-op implementation of the Logger Debugf method.
func (n *nopLogger) Debugf(format string, v ...interface{}) {}
// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger.
type StandardLogger struct {
// standardLogger is a basic implementation of pilosa.Logger based on log.Logger.
type standardLogger struct {
logger *log.Logger
}
func NewStandardLogger(w io.Writer) *StandardLogger {
return &StandardLogger{
func NewStandardLogger(w io.Writer) *standardLogger {
return &standardLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (s *StandardLogger) Printf(format string, v ...interface{}) {
func (s *standardLogger) Printf(format string, v ...interface{}) {
s.logger.Printf(format, v...)
}
func (s *StandardLogger) Debugf(format string, v ...interface{}) {}
func (s *standardLogger) Debugf(format string, v ...interface{}) {}
func (s *StandardLogger) Logger() *log.Logger {
func (s *standardLogger) Logger() *log.Logger {
return s.logger
}
// VerboseLogger is an implementation of pilosa.Logger which includes debug messages.
type VerboseLogger struct {
// verboseLogger is an implementation of pilosa.Logger which includes debug messages.
type verboseLogger struct {
logger *log.Logger
}
func NewVerboseLogger(w io.Writer) *VerboseLogger {
return &VerboseLogger{
func NewVerboseLogger(w io.Writer) *verboseLogger {
return &verboseLogger{
logger: log.New(w, "", log.LstdFlags),
}
}
func (vb *VerboseLogger) Printf(format string, v ...interface{}) {
func (vb *verboseLogger) Printf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Debugf(format string, v ...interface{}) {
func (vb *verboseLogger) Debugf(format string, v ...interface{}) {
vb.logger.Printf(format, v...)
}
func (vb *VerboseLogger) Logger() *log.Logger {
func (vb *verboseLogger) Logger() *log.Logger {
return vb.logger
}

22
lru/lru.go
View file

@ -21,9 +21,9 @@ import "container/list"
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// maxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
maxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
@ -46,7 +46,7 @@ type entry struct {
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
maxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
@ -65,8 +65,8 @@ func (c *Cache) Add(key Key, value interface{}) {
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
if c.maxEntries != 0 && c.ll.Len() > c.maxEntries {
c.removeOldest()
}
}
@ -82,8 +82,8 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) {
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
// remove removes the provided key from the cache.
func (c *Cache) remove(key Key) {
if c.cache == nil {
return
}
@ -92,8 +92,8 @@ func (c *Cache) Remove(key Key) {
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
if c.cache == nil {
return
}
@ -120,8 +120,8 @@ func (c *Cache) Len() int {
return c.ll.Len()
}
// Clear purges all stored items from the cache.
func (c *Cache) Clear() {
// clear purges all stored items from the cache.
func (c *Cache) clear() {
if c.OnEvicted != nil {
for _, e := range c.cache {
kv := e.Value.(*entry)

39
pilosa.go
View file

@ -17,8 +17,6 @@ package pilosa
import (
"errors"
"regexp"
"github.com/pilosa/pilosa/internal"
)
// System errors.
@ -65,15 +63,15 @@ var (
ErrNotImplemented = errors.New("not implemented")
)
// ApiMethodNotAllowedError wraps an error value indicating that a particular
// apiMethodNotAllowedError wraps an error value indicating that a particular
// API method is not allowed in the current cluster state.
type ApiMethodNotAllowedError struct {
type apiMethodNotAllowedError struct {
error
}
// NewApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError.
func NewApiMethodNotAllowedError(err error) ApiMethodNotAllowedError {
return ApiMethodNotAllowedError{err}
// newApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError.
func newApiMethodNotAllowedError(err error) apiMethodNotAllowedError {
return apiMethodNotAllowedError{err}
}
// BadRequestError wraps an error value to signify that a request could not be
@ -95,8 +93,8 @@ type ConflictError struct {
error
}
// NewConflictError returns err wrapped in a ConflictError.
func NewConflictError(err error) ConflictError {
// newConflictError returns err wrapped in a ConflictError.
func newConflictError(err error) ConflictError {
return ConflictError{err}
}
@ -106,8 +104,8 @@ type NotFoundError struct {
error
}
// NewNotFoundError returns err wrapped in a NotFoundError.
func NewNotFoundError(err error) NotFoundError {
// newNotFoundError returns err wrapped in a NotFoundError.
func newNotFoundError(err error) NotFoundError {
return NotFoundError{err}
}
@ -122,23 +120,6 @@ type ColumnAttrSet struct {
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
// EncodeColumnAttrSets converts a into its internal representation.
func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
other := make([]*internal.ColumnAttrSet, len(a))
for i := range a {
other[i] = EncodeColumnAttrSet(a[i])
}
return other
}
// EncodeColumnAttrSet converts set into its internal representation.
func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
return &internal.ColumnAttrSet{
ID: set.ID,
Attrs: encodeAttrs(set.Attrs),
}
}
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
@ -178,7 +159,7 @@ func stringSlicesAreEqual(a, b []string) bool {
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {
if addr == "" {
return DefaultURI(), nil
return defaultURI(), nil
} else {
return NewURIFromAddress(addr)
}

67
pql/ast.go
View file

@ -220,23 +220,6 @@ func (q *Query) WriteCallN() int {
return n
}
// HasKeys returns true if any call in the query uses keys and requires translation to ids.
func (q *Query) HasKeys() bool {
for _, call := range q.Calls {
if call.Args["col"] != nil {
if _, ok := call.Args["col"].(string); ok {
return true
}
}
if call.Args["row"] != nil {
if _, ok := call.Args["row"].(string); ok {
return true
}
}
}
return false
}
// String returns a string representation of the query.
func (q *Query) String() string {
a := make([]string, len(q.Calls))
@ -285,6 +268,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
}
}
// IntArg is for reading the value at key from call.Args as an int64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a unt64 or an int64 and
// then cast to an int64. An error is returned if the value is not an int64 or
// uint64.
func (c *Call) IntArg(key string) (int64, bool, error) {
val, ok := c.Args[key]
if !ok {
return 0, false, nil
}
switch tval := val.(type) {
case int64:
return tval, true, nil
case uint64:
return int64(tval), true, nil
default:
return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval)
}
}
// UintSliceArg reads the value at key from call.Args as a slice of uint64. If
// the key is not in Call.Args, the value of the returned bool will be false,
// and the error will be nil. If the value is a slice of int64 it will convert
@ -309,24 +312,8 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
}
}
// StringArg is for reading the value at key from call.Args as a string. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. An error is returned if the value is not a string.
func (c *Call) StringArg(key string) (string, bool, error) {
val, ok := c.Args[key]
if !ok {
return "", false, nil
}
switch tval := val.(type) {
case string:
return tval, true, nil
default:
return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval)
}
}
// Keys returns a list of argument keys in sorted order.
func (c *Call) Keys() []string {
// keys returns a list of argument keys in sorted order.
func (c *Call) keys() []string {
a := make([]string, 0, len(c.Args))
for k := range c.Args {
a = append(a, k)
@ -382,7 +369,7 @@ func (c *Call) String() string {
}
// Write arguments in key order.
for i, key := range c.Keys() {
for i, key := range c.keys() {
if i > 0 {
buf.WriteString(", ")
}
@ -392,7 +379,7 @@ func (c *Call) String() string {
case *Condition:
fmt.Fprintf(&buf, "%v %s", key, v.String())
default:
fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v))
fmt.Fprintf(&buf, "%v=%s", key, formatValue(v))
}
}
@ -421,7 +408,7 @@ type Condition struct {
// String returns the string representation of the condition.
func (cond *Condition) String() string {
return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value))
return fmt.Sprintf("%s %s", cond.Op.String(), formatValue(cond.Value))
}
// IntSliceValue reads cond.Value as a slice of uint64.
@ -449,7 +436,7 @@ func (cond *Condition) IntSliceValue() ([]int64, error) {
}
}
func FormatValue(v interface{}) string {
func formatValue(v interface{}) string {
switch v := v.(type) {
case string:
return fmt.Sprintf("%q", v)
@ -458,7 +445,7 @@ func FormatValue(v interface{}) string {
case []uint64:
return fmt.Sprintf("%s", joinUint64Slice(v))
case time.Time:
return fmt.Sprintf("\"%s\"", v.Format(TimeFormat))
return fmt.Sprintf("\"%s\"", v.Format(timeFormat))
case *Condition:
return v.String()
default:

14
pql/parser.go
View file

@ -22,19 +22,19 @@ import (
"github.com/pkg/errors"
)
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// timeFormat is the go-style time format used to parse string dates.
const timeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
// parser represents a parser for the PQL language.
type parser struct {
r io.Reader
//scanner *bufScanner
PQL
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
func NewParser(r io.Reader) *parser {
return &parser{
r: r,
// scanner: newBufScanner(r),
}
@ -46,7 +46,7 @@ func ParseString(s string) (*Query, error) {
}
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
func (p *parser) Parse() (*Query, error) {
buf, err := ioutil.ReadAll(p.r)
if err != nil {
return nil, errors.Wrap(err, "reading buffer to parse")

44
roaring/containers.go
View file

@ -14,18 +14,18 @@
package roaring
type SliceContainers struct {
type sliceContainers struct {
keys []uint64
containers []*Container
lastKey uint64
lastContainer *Container
}
func NewSliceContainers() *SliceContainers {
return &SliceContainers{}
func newSliceContainers() *sliceContainers {
return &sliceContainers{}
}
func (sc *SliceContainers) Get(key uint64) *Container {
func (sc *sliceContainers) Get(key uint64) *Container {
i := search64(sc.keys, key)
if i < 0 {
return nil
@ -33,7 +33,7 @@ func (sc *SliceContainers) Get(key uint64) *Container {
return sc.containers[i]
}
func (sc *SliceContainers) Put(key uint64, c *Container) {
func (sc *sliceContainers) Put(key uint64, c *Container) {
i := search64(sc.keys, key)
// If index is negative then there's not an exact match
@ -46,7 +46,7 @@ func (sc *SliceContainers) Put(key uint64, c *Container) {
}
func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
i := search64(sc.keys, key)
if i < 0 {
c := NewContainer()
@ -63,7 +63,7 @@ func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n
}
func (sc *SliceContainers) Remove(key uint64) {
func (sc *sliceContainers) Remove(key uint64) {
i := search64(sc.keys, key)
if i < 0 {
return
@ -72,7 +72,7 @@ func (sc *SliceContainers) Remove(key uint64) {
sc.containers = append(sc.containers[:i], sc.containers[i+1:]...)
}
func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) {
func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) {
sc.keys = append(sc.keys, 0)
copy(sc.keys[i+1:], sc.keys[i:])
sc.keys[i] = key
@ -82,7 +82,7 @@ func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) {
sc.containers[i] = c
}
func (sc *SliceContainers) GetOrCreate(key uint64) *Container {
func (sc *sliceContainers) GetOrCreate(key uint64) *Container {
// Check the last* cache for same container.
if key == sc.lastKey && sc.lastContainer != nil {
return sc.lastContainer
@ -101,8 +101,8 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container {
return sc.lastContainer
}
func (sc *SliceContainers) Clone() Containers {
other := NewSliceContainers()
func (sc *sliceContainers) Clone() Containers {
other := newSliceContainers()
other.keys = make([]uint64, len(sc.keys))
other.containers = make([]*Container, len(sc.containers))
copy(other.keys, sc.keys)
@ -112,19 +112,19 @@ func (sc *SliceContainers) Clone() Containers {
return other
}
func (sc *SliceContainers) Last() (key uint64, c *Container) {
func (sc *sliceContainers) Last() (key uint64, c *Container) {
if len(sc.keys) == 0 {
return 0, nil
}
return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1]
}
func (sc *SliceContainers) Size() int {
func (sc *sliceContainers) Size() int {
return len(sc.keys)
}
func (sc *SliceContainers) Count() uint64 {
func (sc *sliceContainers) Count() uint64 {
n := uint64(0)
for i := range sc.containers {
n += uint64(sc.containers[i].n)
@ -132,14 +132,14 @@ func (sc *SliceContainers) Count() uint64 {
return n
}
func (sc *SliceContainers) Reset() {
func (sc *sliceContainers) Reset() {
sc.keys = sc.keys[:0]
sc.containers = sc.containers[:0]
sc.lastContainer = nil
sc.lastKey = 0
}
func (sc *SliceContainers) seek(key uint64) (int, bool) {
func (sc *sliceContainers) seek(key uint64) (int, bool) {
i := search64(sc.keys, key)
found := true
if i < 0 {
@ -149,19 +149,19 @@ func (sc *SliceContainers) seek(key uint64) (int, bool) {
return i, found
}
func (sc *SliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
i, found := sc.seek(key)
return &SliceIterator{e: sc, i: i}, found
return &sliceIterator{e: sc, i: i}, found
}
type SliceIterator struct {
e *SliceContainers
type sliceIterator struct {
e *sliceContainers
i int
key uint64
value *Container
}
func (si *SliceIterator) Next() bool {
func (si *sliceIterator) Next() bool {
if si.e == nil || si.i > len(si.e.keys)-1 {
return false
}
@ -172,6 +172,6 @@ func (si *SliceIterator) Next() bool {
return true
}
func (si *SliceIterator) Value() (uint64, *Container) {
func (si *sliceIterator) Value() (uint64, *Container) {
return si.key, si.value
}

176
roaring/roaring.go
View file

@ -51,14 +51,14 @@ const (
// bitmapN is the number of values in a container.bitmap.
bitmapN = (1 << 16) / 64
//ContainerArray indicates a container of bit position values
ContainerArray = byte(1)
//containerArray indicates a container of bit position values
containerArray = byte(1)
//ContainerBitmap indicates a container of bits packed in a uint64 array block
ContainerBitmap = byte(2)
//containerBitmap indicates a container of bits packed in a uint64 array block
containerBitmap = byte(2)
//ContainerRun indicates a container of run encoded bits
ContainerRun = byte(3)
//containerRun indicates a container of run encoded bits
containerRun = byte(3)
maxContainerVal = 0xffff
)
@ -117,7 +117,7 @@ type Bitmap struct {
// NewBitmap returns a Bitmap with an initial set of values.
func NewBitmap(a ...uint64) *Bitmap {
b := &Bitmap{
Containers: NewSliceContainers(),
Containers: newSliceContainers(),
}
b.Add(a...)
return b
@ -500,7 +500,7 @@ func (b *Bitmap) Optimize() {
citer, _ := b.Containers.Iterator(0)
for citer.Next() {
_, c := citer.Value()
c.Optimize()
c.optimize()
}
}
@ -657,18 +657,18 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
citer.Next()
_, c := citer.Value()
switch c.containerType {
case ContainerRun:
case containerRun:
c.array = nil
c.bitmap = nil
runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size
case ContainerArray:
case containerArray:
c.runs = nil
c.bitmap = nil
c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]
opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32)
case ContainerBitmap:
case containerBitmap:
c.array = nil
c.runs = nil
c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
@ -725,10 +725,10 @@ func (b *Bitmap) Iterator() *Iterator {
}
// Info returns stats for the bitmap.
func (b *Bitmap) Info() BitmapInfo {
info := BitmapInfo{
func (b *Bitmap) Info() bitmapInfo {
info := bitmapInfo{
OpN: b.opN,
Containers: make([]ContainerInfo, 0, b.Containers.Size()),
Containers: make([]containerInfo, 0, b.Containers.Size()),
}
citer, _ := b.Containers.Iterator(0)
@ -788,10 +788,10 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap {
return result
}
// BitmapInfo represents a point-in-time snapshot of bitmap stats.
type BitmapInfo struct {
// bitmapInfo represents a point-in-time snapshot of bitmap stats.
type bitmapInfo struct {
OpN int
Containers []ContainerInfo
Containers []containerInfo
}
// Iterator represents an iterator over a Bitmap.
@ -987,8 +987,8 @@ func (itr *Iterator) peek() uint64 {
// ArrayMaxSize represents the maximum size of array containers.
const ArrayMaxSize = 4096
// RunMaxSize represents the maximum size of run length encoded containers.
const RunMaxSize = 2048
// runMaxSize represents the maximum size of run length encoded containers.
const runMaxSize = 2048
// Container represents a Container for uint16 integers.
//
@ -1021,7 +1021,7 @@ func (iv interval16) runlen() int {
// newContainer returns a new instance of container.
func NewContainer() *Container {
return &Container{containerType: ContainerArray}
return &Container{containerType: containerArray}
}
// Mapped returns true if the container is mapped directly to a byte slice
@ -1043,17 +1043,17 @@ func (c *Container) Update(containerType byte, n int, mapped bool) {
// isArray returns true if the container is an array container.
func (c *Container) isArray() bool {
return c.containerType == ContainerArray
return c.containerType == containerArray
}
// isBitmap returns true if the container is a bitmap container.
func (c *Container) isBitmap() bool {
return c.containerType == ContainerBitmap
return c.containerType == containerBitmap
}
// isRun returns true if the container is a run-length-encoded container.
func (c *Container) isRun() bool {
return c.containerType == ContainerRun
return c.containerType == containerRun
}
// unmap creates copies of the containers data in the heap.
@ -1066,15 +1066,15 @@ func (c *Container) unmap() {
}
switch c.containerType {
case ContainerArray:
case containerArray:
tmp := make([]uint16, len(c.array))
copy(tmp, c.array)
c.array = tmp
case ContainerBitmap:
case containerBitmap:
tmp := make([]uint64, len(c.bitmap))
copy(tmp, c.bitmap)
c.bitmap = tmp
case ContainerRun:
case containerRun:
tmp := make([]interval16, len(c.runs))
copy(tmp, c.runs)
c.runs = tmp
@ -1315,40 +1315,40 @@ func (c *Container) countRuns() (r int) {
return 0
}
// Optimize converts the container to the type which will take up the least
// optimize converts the container to the type which will take up the least
// amount of space.
func (c *Container) Optimize() {
func (c *Container) optimize() {
if c.n == 0 {
return
}
runs := c.countRuns()
var newType byte
if runs <= RunMaxSize && runs <= c.n/2 {
newType = ContainerRun
if runs <= runMaxSize && runs <= c.n/2 {
newType = containerRun
} else if c.n < ArrayMaxSize {
newType = ContainerArray
newType = containerArray
} else {
newType = ContainerBitmap
newType = containerBitmap
}
// Then convert accordingly.
if c.isArray() {
if newType == ContainerBitmap {
if newType == containerBitmap {
c.arrayToBitmap()
} else if newType == ContainerRun {
} else if newType == containerRun {
c.arrayToRun()
}
} else if c.isBitmap() {
if newType == ContainerArray {
if newType == containerArray {
c.bitmapToArray()
} else if newType == ContainerRun {
} else if newType == containerRun {
c.bitmapToRun()
}
} else if c.isRun() {
if newType == ContainerBitmap {
if newType == containerBitmap {
c.runToBitmap()
} else if newType == ContainerArray {
} else if newType == containerArray {
c.runToArray()
}
}
@ -1487,7 +1487,7 @@ func (c *Container) runMax() uint16 {
// bitmapToArray converts from bitmap format to array format.
func (c *Container) bitmapToArray() {
c.array = make([]uint16, 0, c.n)
c.containerType = ContainerArray
c.containerType = containerArray
// return early if empty
if c.n == 0 {
@ -1510,7 +1510,7 @@ func (c *Container) bitmapToArray() {
// arrayToBitmap converts from array format to bitmap format.
func (c *Container) arrayToBitmap() {
c.bitmap = make([]uint64, bitmapN)
c.containerType = ContainerBitmap
c.containerType = containerBitmap
// return early if empty
if c.n == 0 {
@ -1529,7 +1529,7 @@ func (c *Container) arrayToBitmap() {
// runToBitmap converts from RLE format to bitmap format.
func (c *Container) runToBitmap() {
c.bitmap = make([]uint64, bitmapN)
c.containerType = ContainerBitmap
c.containerType = containerBitmap
// return early if empty
if c.n == 0 {
@ -1551,7 +1551,7 @@ func (c *Container) runToBitmap() {
// bitmapToRun converts from bitmap format to RLE format.
func (c *Container) bitmapToRun() {
c.containerType = ContainerRun
c.containerType = containerRun
// return early if empty
if c.n == 0 {
c.runs = make([]interval16, 0)
@ -1607,7 +1607,7 @@ func (c *Container) bitmapToRun() {
// arrayToRun converts from array format to RLE format.
func (c *Container) arrayToRun() {
c.containerType = ContainerRun
c.containerType = containerRun
// return early if empty
if c.n == 0 {
c.runs = make([]interval16, 0)
@ -1634,7 +1634,7 @@ func (c *Container) arrayToRun() {
// runToArray converts from RLE format to array format.
func (c *Container) runToArray() {
c.containerType = ContainerArray
c.containerType = containerArray
c.array = make([]uint16, 0, c.n)
// return early if empty
@ -1658,13 +1658,13 @@ func (c *Container) Clone() *Container {
other := &Container{n: c.n, containerType: c.containerType}
switch c.containerType {
case ContainerArray:
case containerArray:
other.array = make([]uint16, len(c.array))
copy(other.array, c.array)
case ContainerBitmap:
case containerBitmap:
other.bitmap = make([]uint64, len(c.bitmap))
copy(other.bitmap, c.bitmap)
case ContainerRun:
case containerRun:
other.runs = make([]interval16, len(c.runs))
copy(other.runs, c.runs)
}
@ -1730,8 +1730,8 @@ func (c *Container) size() int {
}
// info returns the current stats about the container.
func (c *Container) info() ContainerInfo {
info := ContainerInfo{N: c.n}
func (c *Container) info() containerInfo {
info := containerInfo{N: c.n}
if c.isArray() {
info.Type = "array"
@ -1787,8 +1787,8 @@ func (c *Container) check() error {
return a
}
// ContainerInfo represents a point-in-time snapshot of container stats.
type ContainerInfo struct {
// containerInfo represents a point-in-time snapshot of container stats.
type containerInfo struct {
Key uint64 // container key
Type string // container type (array, bitmap, or run)
N int // number of bits
@ -1816,7 +1816,7 @@ func flipArray(b *Container) *Container {
}
func flipBitmap(b *Container) *Container {
other := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
other := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
for i, bitmap := range b.bitmap {
other.bitmap[i] = ^bitmap
@ -1977,7 +1977,7 @@ func intersect(a, b *Container) *Container {
}
func intersectArrayArray(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.array[j]
@ -1998,7 +1998,7 @@ func intersectArrayArray(a, b *Container) *Container {
// container. The return is always an array container (since it's guaranteed to
// be low-cardinality)
func intersectArrayRun(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
na, nb := len(a.array), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.runs[j]
@ -2017,7 +2017,7 @@ func intersectArrayRun(a, b *Container) *Container {
// intersectRunRun computes the intersect of two run containers.
func intersectRunRun(a, b *Container) *Container {
output := &Container{containerType: ContainerRun}
output := &Container{containerType: containerRun}
na, nb := len(a.runs), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.runs[i], b.runs[j]
@ -2047,7 +2047,7 @@ func intersectRunRun(a, b *Container) *Container {
}
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -2059,7 +2059,7 @@ func intersectBitmapRun(a, b *Container) *Container {
var output *Container
if b.n < ArrayMaxSize {
// output is array container
output = &Container{containerType: ContainerArray}
output = &Container{containerType: containerArray}
for _, iv := range b.runs {
for i := iv.start; i <= iv.last; i++ {
if a.bitmapContains(i) {
@ -2078,7 +2078,7 @@ func intersectBitmapRun(a, b *Container) *Container {
// the bitmap which are between runs.
output = &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
containerType: containerBitmap,
}
for j := 0; j < len(b.runs); j++ {
vb := b.runs[j]
@ -2119,7 +2119,7 @@ func intersectBitmapRun(a, b *Container) *Container {
}
func intersectArrayBitmap(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
for _, va := range a.array {
bmidx := va / 64
bidx := va % 64
@ -2134,7 +2134,7 @@ func intersectArrayBitmap(a, b *Container) *Container {
}
func intersectBitmapBitmap(a, b *Container) *Container {
output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
for i := range a.bitmap {
v := a.bitmap[i] & b.bitmap[i]
@ -2142,7 +2142,7 @@ func intersectBitmapBitmap(a, b *Container) *Container {
output.n += int(popcount(v))
}
output.Optimize()
output.optimize()
return output
}
@ -2175,7 +2175,7 @@ func union(a, b *Container) *Container {
}
func unionArrayArray(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; ; {
if i >= na && j >= nb {
@ -2211,7 +2211,7 @@ func unionArrayRun(a, b *Container) *Container {
if b.n == maxContainerVal+1 {
return b.Clone()
}
output := &Container{containerType: ContainerRun}
output := &Container{containerType: containerRun}
na, nb := len(a.array), len(b.runs)
var vb interval16
var va uint16
@ -2232,7 +2232,7 @@ func unionArrayRun(a, b *Container) *Container {
}
if output.n < ArrayMaxSize {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -2274,7 +2274,7 @@ func unionRunRun(a, b *Container) *Container {
na, nb := len(a.runs), len(b.runs)
output := &Container{
runs: make([]interval16, 0, na+nb),
containerType: ContainerRun,
containerType: containerRun,
}
var va, vb interval16
for i, j := 0, 0; i < na || j < nb; {
@ -2292,7 +2292,7 @@ func unionRunRun(a, b *Container) *Container {
j++
}
}
if len(output.runs) > RunMaxSize {
if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -2387,7 +2387,7 @@ func (c *Container) equals(c2 *Container) bool {
if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n {
return false
}
if c.containerType == ContainerArray {
if c.containerType == containerArray {
if len(c.array) != len(c2.array) {
return false
}
@ -2396,7 +2396,7 @@ func (c *Container) equals(c2 *Container) bool {
return false
}
}
} else if c.containerType == ContainerBitmap {
} else if c.containerType == containerBitmap {
if len(c.bitmap) != len(c2.bitmap) {
return false
}
@ -2405,7 +2405,7 @@ func (c *Container) equals(c2 *Container) bool {
return false
}
}
} else if c.containerType == ContainerRun {
} else if c.containerType == containerRun {
if len(c.runs) != len(c2.runs) {
return false
}
@ -2434,7 +2434,7 @@ func unionArrayBitmap(a, b *Container) *Container {
func unionBitmapBitmap(a, b *Container) *Container {
output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
containerType: containerBitmap,
}
for i := 0; i < bitmapN; i++ {
@ -2476,7 +2476,7 @@ func difference(a, b *Container) *Container {
// differenceArrayArray computes the difference bween two arrays.
func differenceArrayArray(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na; {
va := a.array[i]
@ -2507,7 +2507,7 @@ func differenceArrayRun(a, b *Container) *Container {
return a.Clone()
}
output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray}
output := &Container{array: make([]uint16, 0, a.n), containerType: containerArray}
// cardinality upper bound: card(A)
i := 0 // array index
@ -2542,7 +2542,7 @@ func differenceArrayRun(a, b *Container) *Container {
// keep all array elements after end of runs
// It's possible that output was converted from array to bitmap in output.add()
// so check container type before proceeding.
if output.containerType == ContainerArray {
if output.containerType == containerArray {
output.array = append(output.array, a.array[i:]...)
// TODO: consider handling container.n mutations in one place
// like we do with container.add().
@ -2575,7 +2575,7 @@ func differenceRunArray(a, b *Container) *Container {
if a.n == 0 || b.n == 0 {
return a.Clone()
}
output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun}
output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: containerRun}
bidx := 0
vb := b.array[bidx]
@ -2621,7 +2621,7 @@ RUNLOOP:
output.n += int(run.last - start + 1)
}
}
output.Optimize()
output.optimize()
return output
}
@ -2631,7 +2631,7 @@ func differenceRunBitmap(a, b *Container) *Container {
if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
return flipBitmap(b)
}
output := &Container{containerType: ContainerRun}
output := &Container{containerType: containerRun}
output.n = a.n
if len(a.runs) == 0 {
return output
@ -2676,7 +2676,7 @@ func differenceRunBitmap(a, b *Container) *Container {
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -2697,7 +2697,7 @@ func differenceRunRun(a, b *Container) *Container {
alen := len(a.runs)
blen := len(b.runs)
output := &Container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else
output := &Container{runs: make([]interval16, 0, alen+blen), containerType: containerRun} // TODO allocate max then truncate? or something else
// cardinality upper bound: sum of number of runs
// each B-run could split an A-run in two, up to len(b.runs) times
@ -2747,7 +2747,7 @@ func differenceRunRun(a, b *Container) *Container {
}
func differenceArrayBitmap(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
for _, va := range a.array {
bmidx := va / 64
bidx := va % 64
@ -2778,7 +2778,7 @@ func differenceBitmapArray(a, b *Container) *Container {
}
func differenceBitmapBitmap(a, b *Container) *Container {
output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
for i := range a.bitmap {
v := a.bitmap[i] & (^b.bitmap[i])
@ -2821,7 +2821,7 @@ func xor(a, b *Container) *Container {
}
func xorArrayArray(a, b *Container) *Container {
output := &Container{containerType: ContainerArray}
output := &Container{containerType: containerArray}
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na || j < nb; {
if i < na && j >= nb {
@ -2861,7 +2861,7 @@ func xorArrayBitmap(a, b *Container) *Container {
// It's possible that output was converted from bitmap to array in output.remove()
// so we only do this conversion if output is still a bitmap container.
if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize {
if output.containerType == containerBitmap && output.count() < ArrayMaxSize {
output.bitmapToArray()
}
@ -2871,7 +2871,7 @@ func xorArrayBitmap(a, b *Container) *Container {
func xorBitmapBitmap(a, b *Container) *Container {
output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: ContainerBitmap,
containerType: containerBitmap,
}
for i := 0; i < bitmapN; i++ {
v := a.bitmap[i] ^ b.bitmap[i]
@ -3079,7 +3079,7 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) {
// xorArrayRun computes the exclusive or of an array and a run container.
func xorArrayRun(a, b *Container) *Container {
output := &Container{containerType: ContainerRun}
output := &Container{containerType: containerRun}
na, nb := len(a.array), len(b.runs)
var vb interval16
var va uint16
@ -3135,7 +3135,7 @@ func xorArrayRun(a, b *Container) *Container {
}
if output.n < ArrayMaxSize {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -3248,7 +3248,7 @@ func xorRunRun(a, b *Container) *Container {
if nb == 0 {
return a.Clone()
}
output := &Container{containerType: ContainerRun}
output := &Container{containerType: containerRun}
lastI, lastJ := -1, -1
@ -3281,7 +3281,7 @@ func xorRunRun(a, b *Container) *Container {
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output
@ -3296,7 +3296,7 @@ func xorBitmapRun(a, b *Container) *Container {
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
output.runToArray()
} else if len(output.runs) > RunMaxSize {
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
}
return output

72
roaring/roaring_helpers_test.go
View file

@ -236,11 +236,11 @@ func doContainer(containerType byte, data interface{}) *Container {
}
switch containerType {
case ContainerArray:
case containerArray:
c.array = data.([]uint16)
case ContainerBitmap:
case containerBitmap:
c.bitmap = data.([]uint64)
case ContainerRun:
case containerRun:
c.runs = data.([]interval16)
}
c.n = c.count()
@ -253,45 +253,45 @@ func setupContainerTests() map[byte]map[string]*Container {
cts := make(map[byte]map[string]*Container)
// array containers
cts[ContainerArray] = map[string]*Container{
"empty": doContainer(ContainerArray, arrayEmpty()),
"full": doContainer(ContainerArray, arrayFull()),
"firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(ContainerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()),
cts[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
}
// bitmap containers
cts[ContainerBitmap] = map[string]*Container{
"empty": doContainer(ContainerBitmap, bitmapEmpty()),
"full": doContainer(ContainerBitmap, bitmapFull()),
"firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()),
cts[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
}
// run containers
cts[ContainerRun] = map[string]*Container{
"empty": doContainer(ContainerRun, runEmpty()),
"full": doContainer(ContainerRun, runFull()),
"firstBitSet": doContainer(ContainerRun, runFirstBitSet()),
"lastBitSet": doContainer(ContainerRun, runLastBitSet()),
"firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(ContainerRun, runLastBitUnset()),
"innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(ContainerRun, runOddBitsSet()),
"evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()),
cts[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
}
return cts

204
roaring/roaring_internal_test.go
View file

@ -33,7 +33,7 @@ func (c *Container) String() string {
}
func TestRunAppendInterval(t *testing.T) {
a := Container{containerType: ContainerRun}
a := Container{containerType: containerRun}
tests := []struct {
base []interval16
app interval16
@ -82,7 +82,7 @@ func TestInterval16RunLen(t *testing.T) {
}
func TestContainerRunAdd(t *testing.T) {
c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
c := Container{runs: make([]interval16, 0), containerType: containerRun}
tests := []struct {
op uint16
exp []interval16
@ -113,7 +113,7 @@ func TestContainerRunAdd(t *testing.T) {
}
func TestContainerRunAdd2(t *testing.T) {
c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
c := Container{runs: make([]interval16, 0), containerType: containerRun}
ret := c.add(0)
if !ret {
t.Fatalf("result of adding new bit should be true: %v", c.runs)
@ -128,7 +128,7 @@ func TestContainerRunAdd2(t *testing.T) {
}
func TestRunCountRange(t *testing.T) {
c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
c := Container{runs: make([]interval16, 0), containerType: containerRun}
cnt := c.runCountRange(2, 9)
if cnt != 0 {
t.Fatalf("should get 0 from empty container, but got: %v", cnt)
@ -181,7 +181,7 @@ func TestRunCountRange(t *testing.T) {
}
func TestRunContains(t *testing.T) {
c := Container{runs: make([]interval16, 0), containerType: ContainerRun}
c := Container{runs: make([]interval16, 0), containerType: containerRun}
if c.runContains(5) {
t.Fatalf("empty run container should not contain 5")
}
@ -203,7 +203,7 @@ func TestRunContains(t *testing.T) {
}
func TestBitmapCountRange(t *testing.T) {
c := Container{containerType: ContainerBitmap}
c := Container{containerType: containerBitmap}
tests := []struct {
start int
end int
@ -229,11 +229,11 @@ func TestBitmapCountRange(t *testing.T) {
func TestIntersectionCountArrayBitmap3(t *testing.T) {
a, b := &Container{}, &Container{}
a.containerType = ContainerBitmap
a.containerType = containerBitmap
a.bitmap = getFullBitmap()
a.n = maxContainerVal + 1
b.containerType = ContainerBitmap
b.containerType = containerBitmap
b.bitmap = getFullBitmap()
b.n = maxContainerVal + 1
res := intersectBitmapBitmap(a, b)
@ -290,9 +290,9 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
for i, test := range tests {
a.array = test.array
a.containerType = ContainerArray
a.containerType = containerArray
b.bitmap = test.bitmap
b.containerType = ContainerBitmap
b.containerType = containerBitmap
ret := intersectionCountArrayBitmap(a, b)
if ret != test.exp {
t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp)
@ -301,7 +301,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
}
func TestRunRemove(t *testing.T) {
c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}
tests := []struct {
op uint16
exp []interval16
@ -335,7 +335,7 @@ func TestRunRemove(t *testing.T) {
}
func TestRunMax(t *testing.T) {
c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}
max := c.max()
if max != 16 {
t.Fatalf("max for %v should be 16", c.runs)
@ -349,8 +349,8 @@ func TestRunMax(t *testing.T) {
}
func TestIntersectionCountArrayRun(t *testing.T) {
a := &Container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
a := &Container{containerType: containerArray, array: []uint16{1, 5, 10, 11, 12}}
b := &Container{containerType: containerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
ret := intersectionCountArrayRun(a, b)
if ret != 3 {
@ -359,16 +359,16 @@ func TestIntersectionCountArrayRun(t *testing.T) {
}
func TestIntersectionCountBitmapRun(t *testing.T) {
a := &Container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
a := &Container{containerType: containerBitmap, bitmap: []uint64{0x8000000000000000}}
b := &Container{containerType: containerRun, runs: []interval16{{start: 63, last: 64}}}
ret := intersectionCountBitmapRun(a, b)
if ret != 1 {
t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret)
}
a = &Container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
a = &Container{containerType: containerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
b = &Container{containerType: containerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
ret = intersectionCountBitmapRun(a, b)
if ret != 14 {
@ -416,8 +416,8 @@ func TestIntersectionCountRunRun(t *testing.T) {
bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
}
for i, test := range tests {
a.containerType = ContainerRun
b.containerType = ContainerRun
a.containerType = containerRun
b.containerType = containerRun
a.runs = test.aruns
b.runs = test.bruns
ret := intersectionCountRunRun(a, b)
@ -458,8 +458,8 @@ func TestIntersectArrayRun(t *testing.T) {
}
for i, test := range tests {
a.containerType = ContainerArray
b.containerType = ContainerRun
a.containerType = containerArray
b.containerType = containerRun
a.array = test.array
b.runs = test.runs
ret := intersectArrayRun(a, b)
@ -516,8 +516,8 @@ func TestIntersectRunRun(t *testing.T) {
},
}
for i, test := range tests {
a.containerType = ContainerRun
b.containerType = ContainerRun
a.containerType = containerRun
b.containerType = containerRun
a.runs = test.aruns
b.runs = test.bruns
ret := intersectRunRun(a, b)
@ -581,8 +581,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
for i, v := range test.exp {
exp[i] = v
}
a.containerType = ContainerBitmap
b.containerType = ContainerRun
a.containerType = containerBitmap
b.containerType = containerRun
ret := intersectBitmapRun(a, b)
if ret.isArray() {
ret.arrayToBitmap()
@ -642,8 +642,8 @@ func TestIntersectBitmapRunArray(t *testing.T) {
a.bitmap[i] = v
}
b.runs = test.runs
a.containerType = ContainerBitmap
b.containerType = ContainerRun
a.containerType = containerBitmap
b.containerType = containerRun
ret := intersectBitmapRun(a, b)
if !reflect.DeepEqual(ret.array, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@ -660,19 +660,19 @@ func TestUnionMixed(t *testing.T) {
// array container
a := &Container{}
a.array = []uint16{1, 4, 5, 7, 10, 11, 12}
a.containerType = ContainerArray
a.containerType = containerArray
a.n = 7
// bitmap container
b := &Container{bitmap: make([]uint64, bitmapN)}
b.bitmap[0] = uint64(0x3)
b.n = 2
b.containerType = ContainerBitmap
b.containerType = containerBitmap
// run container
r := &Container{}
r.runs = []interval16{{start: 5, last: 10}}
r.containerType = ContainerRun
r.containerType = containerRun
r.n = 6
t.Run("various container Unions", func(t *testing.T) {
@ -713,10 +713,10 @@ func TestIntersectMixed(t *testing.T) {
a.runs = []interval16{{start: 5, last: 10}}
a.n = 6
a.containerType = ContainerRun
a.containerType = containerRun
b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
b.n = 7
b.containerType = ContainerArray
b.containerType = containerArray
res := intersect(a, b)
if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) {
t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array)
@ -732,7 +732,7 @@ func TestIntersectMixed(t *testing.T) {
}
c.bitmap = []uint64{0x60}
c.n = 2
c.containerType = ContainerBitmap
c.containerType = containerBitmap
res = intersect(c, a)
if !reflect.DeepEqual(res.array, []uint16{5, 6}) {
@ -762,15 +762,15 @@ func TestDifferenceMixed(t *testing.T) {
a.runs = []interval16{{start: 5, last: 10}}
a.n = a.runCountRange(0, 100)
a.containerType = ContainerRun
a.containerType = containerRun
b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
b.n = len(b.array)
b.containerType = ContainerArray
b.containerType = containerArray
d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
d.n = len(d.array)
d.containerType = ContainerArray
d.containerType = containerArray
res := difference(a, b)
@ -790,7 +790,7 @@ func TestDifferenceMixed(t *testing.T) {
c.bitmap = []uint64{0x64}
c.n = c.countRange(0, 100)
c.containerType = ContainerBitmap
c.containerType = containerBitmap
res = difference(c, a)
if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
@ -885,8 +885,8 @@ func TestUnionRunRun(t *testing.T) {
for i, test := range tests {
a.runs = test.aruns
b.runs = test.bruns
a.containerType = ContainerRun
b.containerType = ContainerRun
a.containerType = containerRun
b.containerType = containerRun
ret := unionRunRun(a, b)
if !reflect.DeepEqual(ret.runs, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@ -927,8 +927,8 @@ func TestUnionArrayRun(t *testing.T) {
for i, test := range tests {
a.array = test.array
b.runs = test.runs
a.containerType = ContainerArray
b.containerType = ContainerRun
a.containerType = containerArray
b.containerType = containerRun
ret := unionArrayRun(a, b)
if !reflect.DeepEqual(ret.array, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@ -937,7 +937,7 @@ func TestUnionArrayRun(t *testing.T) {
}
func TestBitmapSetRange(t *testing.T) {
c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@ -977,7 +977,7 @@ func TestBitmapSetRange(t *testing.T) {
}
func TestArrayToBitmap(t *testing.T) {
a := &Container{containerType: ContainerArray}
a := &Container{containerType: containerArray}
tests := []struct {
array []uint16
exp []uint64
@ -1008,7 +1008,7 @@ func TestArrayToBitmap(t *testing.T) {
}
func TestBitmapToArray(t *testing.T) {
a := &Container{containerType: ContainerBitmap}
a := &Container{containerType: containerBitmap}
tests := []struct {
bitmap []uint64
exp []uint16
@ -1039,7 +1039,7 @@ func TestBitmapToArray(t *testing.T) {
}
func TestRunToBitmap(t *testing.T) {
a := &Container{containerType: ContainerRun}
a := &Container{containerType: containerRun}
tests := []struct {
runs []interval16
exp []uint64
@ -1093,7 +1093,7 @@ func getFullBitmap() []uint64 {
}
func TestBitmapToRun(t *testing.T) {
a := &Container{containerType: ContainerBitmap}
a := &Container{containerType: containerBitmap}
tests := []struct {
bitmap []uint64
exp []interval16
@ -1171,7 +1171,7 @@ func TestBitmapToRun(t *testing.T) {
}
func TestArrayToRun(t *testing.T) {
a := &Container{containerType: ContainerArray}
a := &Container{containerType: containerArray}
tests := []struct {
array []uint16
exp []interval16
@ -1205,7 +1205,7 @@ func TestArrayToRun(t *testing.T) {
}
func TestRunToArray(t *testing.T) {
a := &Container{containerType: ContainerRun}
a := &Container{containerType: containerRun}
tests := []struct {
runs []interval16
exp []uint16
@ -1239,7 +1239,7 @@ func TestRunToArray(t *testing.T) {
}
func TestBitmapZeroRange(t *testing.T) {
c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@ -1283,8 +1283,8 @@ func TestBitmapZeroRange(t *testing.T) {
}
func TestUnionBitmapRun(t *testing.T) {
a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
b := &Container{containerType: containerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -1322,7 +1322,7 @@ func TestUnionBitmapRun(t *testing.T) {
}
func TestBitmapCountRuns(t *testing.T) {
c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
exp int
@ -1372,7 +1372,7 @@ func TestBitmapCountRuns(t *testing.T) {
}
func TestArrayCountRuns(t *testing.T) {
c := &Container{containerType: ContainerArray}
c := &Container{containerType: containerArray}
tests := []struct {
array []uint16
exp int
@ -1413,8 +1413,8 @@ func TestArrayCountRuns(t *testing.T) {
}
func TestDifferenceArrayRun(t *testing.T) {
a := &Container{containerType: ContainerArray}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerArray}
b := &Container{containerType: containerRun}
tests := []struct {
array []uint16
runs []interval16
@ -1439,8 +1439,8 @@ func TestDifferenceArrayRun(t *testing.T) {
}
func TestDifferenceRunArray(t *testing.T) {
a := &Container{containerType: ContainerRun}
b := &Container{containerType: ContainerArray}
a := &Container{containerType: containerRun}
b := &Container{containerType: containerArray}
tests := []struct {
runs []interval16
array []uint16
@ -1520,8 +1520,8 @@ func MakeLastBitSet() []uint64 {
}
func TestDifferenceRunBitmap(t *testing.T) {
a := &Container{containerType: ContainerRun}
b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
a := &Container{containerType: containerRun}
b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
runs []interval16
bitmap []uint64
@ -1583,8 +1583,8 @@ func TestDifferenceRunBitmap(t *testing.T) {
}
func TestDifferenceBitmapRun(t *testing.T) {
a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
b := &Container{containerType: containerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -1666,8 +1666,8 @@ func TestDifferenceBitmapRun(t *testing.T) {
}
func TestDifferenceBitmapArray(t *testing.T) {
b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
a := &Container{containerType: ContainerArray}
b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
a := &Container{containerType: containerArray}
tests := []struct {
bitmap []uint64
array []uint16
@ -1716,8 +1716,8 @@ func TestDifferenceBitmapArray(t *testing.T) {
}
func TestDifferenceBitmapBitmap(t *testing.T) {
a := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
b := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
a := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
b := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
tests := []struct {
abitmap []uint64
bbitmap []uint64
@ -1746,8 +1746,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
}
func TestDifferenceRunRun(t *testing.T) {
a := &Container{containerType: ContainerRun}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerRun}
b := &Container{containerType: containerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@ -1780,7 +1780,7 @@ func TestDifferenceRunRun(t *testing.T) {
}
func TestWriteReadArray(t *testing.T) {
ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray}
ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: containerArray}
ba := NewFileBitmap()
ba.Containers.Put(0, ca)
ba2 := NewFileBitmap()
@ -1800,7 +1800,7 @@ func TestWriteReadArray(t *testing.T) {
func TestWriteReadBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap}
cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: containerBitmap}
for i := 0; i < 129; i++ {
cb.bitmap[i] = 0x5555555555555555
}
@ -1823,7 +1823,7 @@ func TestWriteReadBitmap(t *testing.T) {
func TestWriteReadFullBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap}
cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: containerBitmap}
for i := 0; i < bitmapN; i++ {
cb.bitmap[i] = 0xffffffffffffffff
}
@ -1852,7 +1852,7 @@ func TestWriteReadFullBitmap(t *testing.T) {
}
func TestWriteReadRun(t *testing.T) {
cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun}
cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: containerRun}
br := NewFileBitmap()
br.Containers.Put(0, cr)
br2 := NewFileBitmap()
@ -1877,21 +1877,21 @@ func TestXorArrayRun(t *testing.T) {
exp *Container
}{
{
a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray},
b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12},
a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: containerArray},
b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun},
exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: containerArray, n: 12},
}, {
a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray},
b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12},
a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: containerArray},
b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun},
exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: containerArray, n: 12},
}, {
a: &Container{array: []uint16{65535}, containerType: ContainerArray},
b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun},
exp: &Container{array: []uint16{65534}, containerType: ContainerArray, n: 1},
a: &Container{array: []uint16{65535}, containerType: containerArray},
b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: containerRun},
exp: &Container{array: []uint16{65534}, containerType: containerArray, n: 1},
}, {
a: &Container{array: []uint16{65535}, containerType: ContainerArray},
b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun},
exp: &Container{array: []uint16{}, containerType: ContainerArray, n: 0},
a: &Container{array: []uint16{65535}, containerType: containerArray},
b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: containerRun},
exp: &Container{array: []uint16{}, containerType: containerArray, n: 0},
},
}
@ -1912,8 +1912,8 @@ func TestXorArrayRun(t *testing.T) {
//special case that didn't fit the xorrunrun table testing below.
func TestXorRunRun1(t *testing.T) {
a := &Container{containerType: ContainerRun}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerRun}
b := &Container{containerType: containerRun}
a.runs = []interval16{{start: 4, last: 10}}
b.runs = []interval16{{start: 5, last: 10}}
ret := xorRunRun(a, b)
@ -1927,8 +1927,8 @@ func TestXorRunRun1(t *testing.T) {
}
func TestXorRunRun(t *testing.T) {
a := &Container{containerType: ContainerRun}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerRun}
b := &Container{containerType: containerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@ -2025,7 +2025,7 @@ func TestXorRunRun(t *testing.T) {
}
func TestBitmapXorRange(t *testing.T) {
c := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
c := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
tests := []struct {
bitmap []uint64
start uint64
@ -2093,8 +2093,8 @@ func TestBitmapXorRange(t *testing.T) {
}
func TestXorBitmapRun(t *testing.T) {
a := &Container{containerType: ContainerBitmap}
b := &Container{containerType: ContainerRun}
a := &Container{containerType: containerBitmap}
b := &Container{containerType: containerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -2546,8 +2546,8 @@ func TestSearch64(t *testing.T) {
}
func TestIntersectArrayBitmap(t *testing.T) {
a, b := &Container{containerType: ContainerArray}, &Container{
containerType: ContainerBitmap,
a, b := &Container{containerType: containerArray}, &Container{
containerType: containerBitmap,
bitmap: make([]uint64, bitmapN),
}
tests := []struct {
@ -2594,11 +2594,11 @@ func TestIntersectArrayBitmap(t *testing.T) {
for i, test := range tests {
a.array = test.array
a.containerType = ContainerArray
a.containerType = containerArray
for i, bmval := range test.bitmap {
b.bitmap[i] = bmval
}
b.containerType = ContainerBitmap
b.containerType = containerBitmap
ret := intersectArrayBitmap(a, b).array
if len(ret) == 0 && len(test.exp) == 0 {
continue
@ -2722,13 +2722,13 @@ func TestContainerCombinations(t *testing.T) {
cts := setupContainerTests()
containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun}
containerTypes := []byte{containerArray, containerBitmap, containerRun}
// map used for a more descriptive print
cm := map[byte]string{
ContainerArray: "array",
ContainerBitmap: "bitmap",
ContainerRun: "run",
containerArray: "array",
containerBitmap: "bitmap",
containerRun: "run",
}
testOps := []testOp{
@ -3198,7 +3198,7 @@ func TestContainerCombinations(t *testing.T) {
// Convert to all container types and check result.
for _, ct := range containerTypes {
clone := ret.Clone()
if ct == ContainerArray {
if ct == containerArray {
if clone.isBitmap() {
clone.bitmapToArray()
} else if clone.isRun() {
@ -3212,7 +3212,7 @@ func TestContainerCombinations(t *testing.T) {
if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) {
t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array)
}
} else if ct == ContainerBitmap {
} else if ct == containerBitmap {
if clone.isArray() {
clone.arrayToBitmap()
} else if clone.isRun() {
@ -3224,7 +3224,7 @@ func TestContainerCombinations(t *testing.T) {
if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) {
t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap)
}
} else if ct == ContainerRun {
} else if ct == containerRun {
if clone.isArray() {
clone.arrayToRun()
} else if clone.isBitmap() {

109
row.go
View file

@ -18,14 +18,13 @@ import (
"encoding/json"
"sort"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/roaring"
)
// Row is a set of integers (the associated columns), and attributes which are
// arbitrary key/value pairs storing metadata about what the row represents.
type Row struct {
segments []RowSegment
segments []rowSegment
// String keys translated to/from segment columns.
Keys []string
@ -45,7 +44,7 @@ func NewRow(columns ...uint64) *Row {
// Merge merges data from other into r.
func (r *Row) Merge(other *Row) {
var segments []RowSegment
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
@ -64,11 +63,11 @@ func (r *Row) Merge(other *Row) {
}
r.segments = segments
r.InvalidateCount()
r.invalidateCount()
}
// IntersectionCount returns the number of intersections between r and other.
func (r *Row) IntersectionCount(other *Row) uint64 {
// intersectionCount returns the number of intersections between r and other.
func (r *Row) intersectionCount(other *Row) uint64 {
var n uint64
itr := newMergeSegmentIterator(r.segments, other.segments)
@ -83,9 +82,9 @@ func (r *Row) IntersectionCount(other *Row) uint64 {
return n
}
// Intersect returns the itersection of r and other.
func (r *Row) Intersect(other *Row) *Row {
var segments []RowSegment
// intersect returns the itersection of r and other.
func (r *Row) intersect(other *Row) *Row {
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
@ -101,7 +100,7 @@ func (r *Row) Intersect(other *Row) *Row {
// Xor returns the xor of r and other.
func (r *Row) Xor(other *Row) *Row {
var segments []RowSegment
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
@ -121,7 +120,7 @@ func (r *Row) Xor(other *Row) *Row {
// Union returns the bitwise union of r and other.
func (r *Row) Union(other *Row) *Row {
var segments []RowSegment
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
@ -139,7 +138,7 @@ func (r *Row) Union(other *Row) *Row {
// Difference returns the diff of r and other.
func (r *Row) Difference(other *Row) *Row {
var segments []RowSegment
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
@ -160,8 +159,8 @@ func (r *Row) SetBit(i uint64) (changed bool) {
return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i)
}
// ClearBit clears the i-th column of the row.
func (r *Row) ClearBit(i uint64) (changed bool) {
// clearBit clears the i-th column of the row.
func (r *Row) clearBit(i uint64) (changed bool) {
s := r.segment(i / ShardWidth)
if s == nil {
return false
@ -170,13 +169,13 @@ func (r *Row) ClearBit(i uint64) (changed bool) {
}
// Segments returns a list of all segments in the row.
func (r *Row) Segments() []RowSegment {
func (r *Row) Segments() []rowSegment {
return r.segments
}
// segment returns a segment for a given shard.
// Returns nil if segment does not exist.
func (r *Row) segment(shard uint64) *RowSegment {
func (r *Row) segment(shard uint64) *rowSegment {
if i := sort.Search(len(r.segments), func(i int) bool {
return r.segments[i].shard >= shard
}); i < len(r.segments) && r.segments[i].shard == shard {
@ -185,7 +184,7 @@ func (r *Row) segment(shard uint64) *RowSegment {
return nil
}
func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment {
func (r *Row) createSegmentIfNotExists(shard uint64) *rowSegment {
i := sort.Search(len(r.segments), func(i int) bool {
return r.segments[i].shard >= shard
})
@ -196,11 +195,11 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment {
}
// Insert new segment.
r.segments = append(r.segments, RowSegment{data: *roaring.NewBitmap()})
r.segments = append(r.segments, rowSegment{data: *roaring.NewBitmap()})
if i < len(r.segments) {
copy(r.segments[i+1:], r.segments[i:])
}
r.segments[i] = RowSegment{
r.segments[i] = rowSegment{
data: *roaring.NewBitmap(),
shard: shard,
writable: true,
@ -209,8 +208,8 @@ func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment {
return &r.segments[i]
}
// InvalidateCount updates the cached count in the row.
func (r *Row) InvalidateCount() {
// invalidateCount updates the cached count in the row.
func (r *Row) invalidateCount() {
for i := range r.segments {
r.segments[i].InvalidateCount()
}
@ -252,36 +251,10 @@ func (r *Row) Columns() []uint64 {
return a
}
// EncodeRow converts r into its internal representation.
func EncodeRow(r *Row) *internal.Row {
if r == nil {
return nil
}
return &internal.Row{
Columns: r.Columns(),
Attrs: encodeAttrs(r.Attrs),
}
}
// DecodeRow converts r from its internal representation.
func DecodeRow(pr *internal.Row) *Row {
if pr == nil {
return nil
}
r := NewRow()
r.Attrs = decodeAttrs(pr.Attrs)
for _, v := range pr.Columns {
r.SetBit(v)
}
return r
}
// RowSegment holds a subset of a row.
// rowSegment holds a subset of a row.
// This could point to a mmapped roaring bitmap or an in-memory bitmap. The
// width of the segment will always match the shard width.
type RowSegment struct {
type rowSegment struct {
// Shard this segment belongs to
shard uint64
@ -297,7 +270,7 @@ type RowSegment struct {
// Merge adds chunks from other to s.
// Chunks in s are overwritten if they exist in other.
func (s *RowSegment) Merge(other *RowSegment) {
func (s *rowSegment) Merge(other *rowSegment) {
s.ensureWritable()
itr := other.data.Iterator()
@ -307,15 +280,15 @@ func (s *RowSegment) Merge(other *RowSegment) {
}
// IntersectionCount returns the number of intersections between s and other.
func (s *RowSegment) IntersectionCount(other *RowSegment) uint64 {
func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
return s.data.IntersectionCount(&other.data)
}
// Intersect returns the itersection of s and other.
func (s *RowSegment) Intersect(other *RowSegment) *RowSegment {
func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
data := s.data.Intersect(&other.data)
return &RowSegment{
return &rowSegment{
data: *data,
shard: s.shard,
n: data.Count(),
@ -323,10 +296,10 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment {
}
// Union returns the bitwise union of s and other.
func (s *RowSegment) Union(other *RowSegment) *RowSegment {
func (s *rowSegment) Union(other *rowSegment) *rowSegment {
data := s.data.Union(&other.data)
return &RowSegment{
return &rowSegment{
data: *data,
shard: s.shard,
n: data.Count(),
@ -334,10 +307,10 @@ func (s *RowSegment) Union(other *RowSegment) *RowSegment {
}
// Difference returns the diff of s and other.
func (s *RowSegment) Difference(other *RowSegment) *RowSegment {
func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
data := s.data.Difference(&other.data)
return &RowSegment{
return &rowSegment{
data: *data,
shard: s.shard,
n: data.Count(),
@ -345,10 +318,10 @@ func (s *RowSegment) Difference(other *RowSegment) *RowSegment {
}
// Xor returns the xor of s and other.
func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
data := s.data.Xor(&other.data)
return &RowSegment{
return &rowSegment{
data: *data,
shard: s.shard,
n: data.Count(),
@ -356,7 +329,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
}
// SetBit sets the i-th column of the row.
func (s *RowSegment) SetBit(i uint64) (changed bool) {
func (s *rowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()
changed, _ = s.data.Add(i)
if changed {
@ -366,7 +339,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) {
}
// ClearBit clears the i-th column of the row.
func (s *RowSegment) ClearBit(i uint64) (changed bool) {
func (s *rowSegment) ClearBit(i uint64) (changed bool) {
s.ensureWritable()
changed, _ = s.data.Remove(i)
@ -377,12 +350,12 @@ func (s *RowSegment) ClearBit(i uint64) (changed bool) {
}
// InvalidateCount updates the cached count in the row.
func (s *RowSegment) InvalidateCount() {
func (s *rowSegment) InvalidateCount() {
s.n = s.data.Count()
}
// Columns returns a list of all columns set in the segment.
func (s *RowSegment) Columns() []uint64 {
func (s *rowSegment) Columns() []uint64 {
a := make([]uint64, 0, s.Count())
itr := s.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
@ -392,10 +365,10 @@ func (s *RowSegment) Columns() []uint64 {
}
// Count returns the number of set columns in the row.
func (s *RowSegment) Count() uint64 { return s.n }
func (s *rowSegment) Count() uint64 { return s.n }
// ensureWritable clones the segment if it is pointing to non-writable data.
func (s *RowSegment) ensureWritable() {
func (s *rowSegment) ensureWritable() {
if s.writable {
return
}
@ -406,16 +379,16 @@ func (s *RowSegment) ensureWritable() {
// mergeSegmentIterator produces an iterator that loops through two sets of segments.
type mergeSegmentIterator struct {
a0, a1 []RowSegment
a0, a1 []rowSegment
}
// newMergeSegmentIterator returns a new instance of mergeSegmentIterator.
func newMergeSegmentIterator(a0, a1 []RowSegment) mergeSegmentIterator {
func newMergeSegmentIterator(a0, a1 []rowSegment) mergeSegmentIterator {
return mergeSegmentIterator{a0: a0, a1: a1}
}
// next returns the next set of segments.
func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) {
func (itr *mergeSegmentIterator) next() (s0, s1 *rowSegment) {
// Find current segments.
if len(itr.a0) > 0 {
s0 = &itr.a0[0]

98
server.go
View file

@ -27,8 +27,6 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -52,10 +50,11 @@ type Server struct {
holder *Holder
cluster *cluster
translateFile *TranslateFile
diagnostics *DiagnosticsCollector
diagnostics *diagnosticsCollector
executor *executor
hosts []string
clusterDisabled bool
serializer Serializer
// External
systemInfo SystemInfo
@ -202,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption {
}
}
func OptServerSerializer(ser Serializer) ServerOption {
return func(s *Server) error {
s.serializer = ser
return nil
}
}
func OptServerIsCoordinator(is bool) ServerOption {
return func(s *Server) error {
s.isCoordinator = is
@ -229,8 +235,8 @@ func NewServer(opts ...ServerOption) (*Server, error) {
closing: make(chan struct{}),
cluster: newCluster(),
holder: NewHolder(),
diagnostics: NewDiagnosticsCollector(defaultDiagnosticServer),
systemInfo: NewNopSystemInfo(),
diagnostics: newDiagnosticsCollector(defaultDiagnosticServer),
systemInfo: newNopSystemInfo(),
gcNotifier: NopGCNotifier,
@ -331,7 +337,7 @@ func (s *Server) Open() error {
if err := s.holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.cluster.setNodeState(NodeStateReady); err != nil {
if err := s.cluster.setNodeState(nodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
@ -431,40 +437,40 @@ func (s *Server) monitorAntiEntropy() {
}
// receiveMessage represents an implementation of BroadcastHandler.
func (s *Server) receiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.CreateShardMessage:
func (s *Server) receiveMessage(m Message) error {
switch obj := m.(type) {
case *CreateShardMessage:
idx := s.holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
idx.setRemoteMaxShard(obj.Shard)
case *internal.CreateIndexMessage:
case *CreateIndexMessage:
opt := IndexOptions{}
_, err := s.holder.CreateIndex(obj.Index, opt)
if err != nil {
return err
}
case *internal.DeleteIndexMessage:
case *DeleteIndexMessage:
if err := s.holder.DeleteIndex(obj.Index); err != nil {
return err
}
case *internal.CreateFieldMessage:
case *CreateFieldMessage:
idx := s.holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := decodeFieldOptions(obj.Meta)
opt := obj.Meta
_, err := idx.createField(obj.Field, *opt)
if err != nil {
return err
}
case *internal.DeleteFieldMessage:
case *DeleteFieldMessage:
idx := s.holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateViewMessage:
case *CreateViewMessage:
f := s.holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Field not found: %s", obj.Field)
@ -473,7 +479,7 @@ func (s *Server) receiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.DeleteViewMessage:
case *DeleteViewMessage:
f := s.holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Field not found: %s", obj.Field)
@ -482,44 +488,49 @@ func (s *Server) receiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.ClusterStatus:
case *ClusterStatus:
err := s.cluster.mergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
case *ResizeInstruction:
err := s.cluster.followResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
case *ResizeInstructionComplete:
err := s.cluster.markResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.cluster.setCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.cluster.updateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
case *SetCoordinatorMessage:
s.cluster.setCoordinator(obj.New)
case *UpdateCoordinatorMessage:
s.cluster.updateCoordinator(obj.New)
case *NodeStateMessage:
err := s.cluster.receiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
case *internal.RecalculateCaches:
s.holder.RecalculateCaches()
case *internal.NodeEventMessage:
s.cluster.ReceiveEvent(DecodeNodeEvent(obj))
case *internal.NodeStatus:
s.handleRemoteStatus(pb)
case *RecalculateCaches:
s.holder.recalculateCaches()
case *NodeEvent:
s.cluster.ReceiveEvent(obj)
case *NodeStatus:
s.handleRemoteStatus(obj)
}
return nil
}
// SendSync represents an implementation of Broadcaster.
func (s *Server) SendSync(pb proto.Message) error {
func (s *Server) SendSync(m Message) error {
var eg errgroup.Group
msg, err := s.serializer.Marshal(m)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
msg = append([]byte{getMessageType(m)}, msg...)
for _, node := range s.cluster.Nodes {
node := node
s.logger.Printf("SendSync to: %s", node.URI)
@ -529,7 +540,7 @@ func (s *Server) SendSync(pb proto.Message) error {
}
eg.Go(func() error {
return s.defaultClient.SendMessage(context.Background(), &node.URI, pb)
return s.defaultClient.SendMessage(context.Background(), &node.URI, msg)
})
}
@ -537,14 +548,19 @@ func (s *Server) SendSync(pb proto.Message) error {
}
// SendAsync represents an implementation of Broadcaster.
func (s *Server) SendAsync(pb proto.Message) error {
func (s *Server) SendAsync(m Message) error {
return ErrNotImplemented
}
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
func (s *Server) SendTo(to *Node, m Message) error {
s.logger.Printf("SendTo: %s", to.URI)
return s.defaultClient.SendMessage(context.Background(), &to.URI, pb)
msg, err := s.serializer.Marshal(m)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
msg = append([]byte{getMessageType(m)}, msg...)
return s.defaultClient.SendMessage(context.Background(), &to.URI, msg)
}
// node returns the pilosa.node object. It is used by membership protocols to
@ -554,7 +570,7 @@ func (s *Server) node() Node {
}
// handleRemoteStatus receives incoming NodeStatus from remote nodes.
func (s *Server) handleRemoteStatus(pb proto.Message) {
func (s *Server) handleRemoteStatus(pb Message) {
// Ignore NodeStatus messages until the cluster is in a Normal state.
if s.cluster.State() != ClusterStateNormal {
return
@ -564,16 +580,16 @@ func (s *Server) handleRemoteStatus(pb proto.Message) {
// Make sure the holder has opened.
<-s.holder.opened
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
err := s.mergeRemoteStatus(pb.(*NodeStatus))
if err != nil {
s.logger.Printf("merge remote status: %s", err)
}
}()
}
func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
// Ignore status updates from self.
if s.nodeID == DecodeNode(ns.Node).ID {
if s.nodeID == ns.Node.ID {
return nil
}
@ -584,7 +600,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// Sync maxShards.
oldmaxshards := s.holder.maxShards()
for index, newMax := range ns.MaxShards.Standard {
for index, newMax := range ns.MaxShards {
localIndex := s.holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to shard creation
@ -613,7 +629,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.uri.host)
s.diagnostics.Set("Host", s.uri.Host)
s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())

7
server/config.go
View file

@ -21,13 +21,6 @@ import (
"github.com/pilosa/pilosa/toml"
)
// Cluster types.
const (
ClusterNone = ""
ClusterStatic = "static"
ClusterGossip = "gossip"
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)

4
server/default.go
View file

@ -20,5 +20,5 @@ package server
import "time"
// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics.
const DefaultDiagnosticsInterval = time.Duration(0)
// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics.
const defaultDiagnosticsInterval = time.Duration(0)

80
server/handler_test.go
View file

@ -27,10 +27,8 @@ import (
gohttp "net/http"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Shards args protobuf", func(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
Query: "Count(Row(f0=30))",
Shards: []uint64{0, 1},
})
@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 3 {
t.Fatalf("unexpected n: %d", n)
} else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 {
t.Fatalf("unexpected response type: %#v", resp.Results[0])
}
})
@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
} else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
})
t.Run("Row columnattrs protobuf", func(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
Query: "Row(f0=30)",
ColumnAttrs: true,
})
@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
} else if _, ok := resp.Results[0].(*pilosa.Row); !ok {
t.Fatalf("unexpected response type: %#v", resp.Results[0])
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
if a := resp.ColumnAttrSets; len(a) != 2 {
@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if a[0].Attrs["x"] != "y" {
t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"])
}
})
@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
} else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
})
@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: field not found` {
} else if s := resp.Err.Error(); s != `executing: field not found` {
t.Fatalf("unexpected error: %s", s)
}
})

4
server/release.go
View file

@ -20,5 +20,5 @@ package server
import "time"
// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics.
const DefaultDiagnosticsInterval = 1 * time.Hour
// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics.
const defaultDiagnosticsInterval = 1 * time.Hour

28
server/server.go
View file

@ -35,6 +35,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
@ -123,7 +124,7 @@ func (m *Command) Start() (err error) {
}
// SetupNetworking
err = m.SetupNetworking()
err = m.setupNetworking()
if err != nil {
return errors.Wrap(err, "setting up networking")
}
@ -202,7 +203,7 @@ func (m *Command) SetupServer() error {
// Setup TLS
var TLSConfig *tls.Config
if uri.Scheme() == "https" {
if uri.Scheme == "https" {
if m.Config.TLS.CertificatePath == "" {
return errors.New("certificate path is required for TLS sockets")
}
@ -221,10 +222,10 @@ func (m *Command) SetupServer() error {
diagnosticsInterval := time.Duration(0)
if m.Config.Metric.Diagnostics {
diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval)
diagnosticsInterval = time.Duration(defaultDiagnosticsInterval)
}
statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
if err != nil {
return errors.Wrap(err, "new stats client")
}
@ -235,7 +236,7 @@ func (m *Command) SetupServer() error {
}
// If port is 0, get auto-allocated port from listener
if uri.Port() == 0 {
if uri.Port == 0 {
uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
}
@ -271,6 +272,7 @@ func (m *Command) SetupServer() error {
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
pilosa.OptServerSerializer(proto.Serializer{}),
coordinatorOpt,
}
@ -297,8 +299,8 @@ func (m *Command) SetupServer() error {
}
// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {
// setupNetworking sets up internode communication based on the configuration.
func (m *Command) setupNetworking() error {
if m.Config.Cluster.Disabled {
return nil
}
@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error {
}
// get the host portion of addr to use for binding
gossipHost := m.API.Node().URI.Host()
gossipHost := m.API.Node().URI.Host
m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
if err != nil {
return errors.Wrap(err, "getting transport")
@ -349,8 +351,8 @@ func (m *Command) Close() error {
return nil
}
// NewStatsClient creates a stats client from the config
func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
// newStatsClient creates a stats client from the config
func newStatsClient(name string, host string) (pilosa.StatsClient, error) {
switch name {
case "expvar":
return pilosa.NewExpvarStatsClient(), nil
@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
// getListener gets a net.Listener based on the config.
func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
// If bind URI has the https scheme, enable TLS
if uri.Scheme() == "https" && tlsconf != nil {
if uri.Scheme == "https" && tlsconf != nil {
ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
if err != nil {
return nil, errors.Wrap(err, "tls.Listener")
}
} else if uri.Scheme() == "http" {
} else if uri.Scheme == "http" {
// Open HTTP listener to determine port (if specified as :0).
ln, err = net.Listen("tcp", uri.HostPort())
if err != nil {
return nil, errors.Wrap(err, "net.Listen")
}
} else {
return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme)
}
return ln, nil

32
stats.go
View file

@ -82,8 +82,8 @@ func (c *nopStatsClient) SetLogger(logger Logger)
func (c *nopStatsClient) Open() {}
func (c *nopStatsClient) Close() error { return nil }
// ExpvarStatsClient writes stats out to expvars.
type ExpvarStatsClient struct {
// expvarStatsClient writes stats out to expvars.
type expvarStatsClient struct {
mu sync.Mutex
m *expvar.Map
tags []string
@ -91,41 +91,41 @@ type ExpvarStatsClient struct {
// NewExpvarStatsClient returns a new instance of ExpvarStatsClient.
// This client points at the root of the expvar index map.
func NewExpvarStatsClient() *ExpvarStatsClient {
return &ExpvarStatsClient{
func NewExpvarStatsClient() *expvarStatsClient {
return &expvarStatsClient{
m: Expvar,
}
}
// Tags returns a sorted list of tags on the client.
func (c *ExpvarStatsClient) Tags() []string {
func (c *expvarStatsClient) Tags() []string {
return nil
}
// WithTags returns a new client with additional tags appended.
func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient {
func (c *expvarStatsClient) WithTags(tags ...string) StatsClient {
m := &expvar.Map{}
m.Init()
c.m.Set(strings.Join(tags, ","), m)
return &ExpvarStatsClient{
return &expvarStatsClient{
m: m,
tags: unionStringSlice(c.tags, tags),
}
}
// Count tracks the number of times something occurs.
func (c *ExpvarStatsClient) Count(name string, value int64, rate float64) {
func (c *expvarStatsClient) Count(name string, value int64, rate float64) {
c.m.Add(name, value)
}
// CountWithCustomTags Tracks the number of times something occurs per second with custom tags
func (c *ExpvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
c.m.Add(name, value)
}
// Gauge sets the value of a metric.
func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) {
func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) {
var f expvar.Float
f.Set(value)
c.m.Set(name, &f)
@ -133,19 +133,19 @@ func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) {
// Histogram tracks statistical distribution of a metric.
// This works the same as gauge for this client.
func (c *ExpvarStatsClient) Histogram(name string, value float64, rate float64) {
func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) {
c.Gauge(name, value, rate)
}
// Set tracks number of unique elements.
func (c *ExpvarStatsClient) Set(name string, value string, rate float64) {
func (c *expvarStatsClient) Set(name string, value string, rate float64) {
var s expvar.String
s.Set(value)
c.m.Set(name, &s)
}
// Timing tracks timing information for a metric.
func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float64) {
func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) {
c.mu.Lock()
d, _ := c.m.Get(name).(time.Duration)
c.m.Set(name, d+value)
@ -153,14 +153,14 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6
}
// SetLogger has no logger.
func (c *ExpvarStatsClient) SetLogger(logger Logger) {
func (c *expvarStatsClient) SetLogger(logger Logger) {
}
// Open no-op.
func (c *ExpvarStatsClient) Open() {}
func (c *expvarStatsClient) Open() {}
// Close no-op.
func (c *ExpvarStatsClient) Close() error { return nil }
func (c *expvarStatsClient) Close() error { return nil }
// MultiStatsClient joins multiple stats clients together.
type MultiStatsClient []StatsClient

56
statsd/statsd.go
View file

@ -26,52 +26,52 @@ import (
// statsD defailt host is "127.0.0.1:8125"
const (
// Prefix is appended to each metric event name
Prefix = "pilosa."
// prefix is appended to each metric event name
prefix = "pilosa."
// BufferLen Stats lient buffer size.
BufferLen = 1024
// bufferLen Stats lient buffer size.
bufferLen = 1024
)
// Ensure client implements interface.
var _ pilosa.StatsClient = &StatsClient{}
var _ pilosa.StatsClient = &statsClient{}
// StatsClient represents a StatsD implementation of pilosa.StatsClient.
type StatsClient struct {
// statsClient represents a StatsD implementation of pilosa.statsClient.
type statsClient struct {
client *statsd.Client
tags []string
logger pilosa.Logger
}
// NewStatsClient returns a new instance of StatsClient.
func NewStatsClient(host string) (*StatsClient, error) {
c, err := statsd.NewBuffered(host, BufferLen)
func NewStatsClient(host string) (*statsClient, error) {
c, err := statsd.NewBuffered(host, bufferLen)
if err != nil {
return nil, err
}
return &StatsClient{
return &statsClient{
client: c,
logger: pilosa.NopLogger,
}, nil
}
// Open no-op
func (c *StatsClient) Open() {}
func (c *statsClient) Open() {}
// Close closes the connection to the agent.
func (c *StatsClient) Close() error {
func (c *statsClient) Close() error {
return c.client.Close()
}
// Tags returns a sorted list of tags on the client.
func (c *StatsClient) Tags() []string {
func (c *statsClient) Tags() []string {
return c.tags
}
// WithTags returns a new client with additional tags appended.
func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
return &StatsClient{
func (c *statsClient) WithTags(tags ...string) pilosa.StatsClient {
return &statsClient{
client: c.client,
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
@ -79,50 +79,50 @@ func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
}
// Count tracks the number of times something occurs per second.
func (c *StatsClient) Count(name string, value int64, rate float64) {
if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil {
func (c *statsClient) Count(name string, value int64, rate float64) {
if err := c.client.Count(prefix+name, value, c.tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Count error: %s", err)
}
}
// CountWithCustomTags tracks the number of times something occurs per second with custom tags.
func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
tags := append(c.tags, t...)
if err := c.client.Count(Prefix+name, value, tags, rate); err != nil {
if err := c.client.Count(prefix+name, value, tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Count error: %s", err)
}
}
// Gauge sets the value of a metric.
func (c *StatsClient) Gauge(name string, value float64, rate float64) {
if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil {
func (c *statsClient) Gauge(name string, value float64, rate float64) {
if err := c.client.Gauge(prefix+name, value, c.tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Gauge error: %s", err)
}
}
// Histogram tracks statistical distribution of a metric.
func (c *StatsClient) Histogram(name string, value float64, rate float64) {
if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil {
func (c *statsClient) Histogram(name string, value float64, rate float64) {
if err := c.client.Histogram(prefix+name, value, c.tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Histogram error: %s", err)
}
}
// Set tracks number of unique elements.
func (c *StatsClient) Set(name string, value string, rate float64) {
if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil {
func (c *statsClient) Set(name string, value string, rate float64) {
if err := c.client.Set(prefix+name, value, c.tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Set error: %s", err)
}
}
// Timing tracks timing information for a metric.
func (c *StatsClient) Timing(name string, value time.Duration, rate float64) {
if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil {
func (c *statsClient) Timing(name string, value time.Duration, rate float64) {
if err := c.client.Timing(prefix+name, value, c.tags, rate); err != nil {
c.logger.Printf("statsd.StatsClient.Timing error: %s", err)
}
}
// SetLogger sets the logger for client.
func (c *StatsClient) SetLogger(logger pilosa.Logger) {
func (c *statsClient) SetLogger(logger pilosa.Logger) {
c.logger = logger
}

24
test/field.go
View file

@ -27,8 +27,8 @@ type Field struct {
*pilosa.Field
}
// NewField returns a new instance of Field d/0.
func NewField(opts pilosa.FieldOption) *Field {
// newField returns a new instance of Field d/0.
func newField(opts pilosa.FieldOption) *Field {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
@ -40,23 +40,23 @@ func NewField(opts pilosa.FieldOption) *Field {
return &Field{Field: field}
}
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(opts pilosa.FieldOption) *Field {
f := NewField(opts)
// mustOpenField returns a new, opened field at a temporary path. Panic on error.
func mustOpenField(opts pilosa.FieldOption) *Field {
f := newField(opts)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the field and removes the underlying data.
func (f *Field) Close() error {
// close closes the field and removes the underlying data.
func (f *Field) close() error {
defer os.RemoveAll(f.Path())
return f.Field.Close()
}
// Reopen closes the index and reopens it.
func (f *Field) Reopen() error {
// reopen closes the index and reopens it.
func (f *Field) reopen() error {
var err error
if err := f.Field.Close(); err != nil {
return err
@ -76,8 +76,8 @@ func (f *Field) Reopen() error {
// Ensure field can set its cache
func TestField_SetCacheSize(t *testing.T) {
f := MustOpenField(pilosa.OptFieldTypeDefault())
defer f.Close()
f := mustOpenField(pilosa.OptFieldTypeDefault())
defer f.close()
cacheSize := uint32(100)
// Set & retrieve field cache size.
@ -88,7 +88,7 @@ func TestField_SetCacheSize(t *testing.T) {
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected field cache size (reopen): %d", q)

9
test/holder.go
View file

@ -81,15 +81,6 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
return &Index{Index: idx}
}
// MustCreateFieldIfNotExists returns a given field. Panic on error.
func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
return f
}
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})

6
test/index.go
View file

@ -26,8 +26,8 @@ type Index struct {
*pilosa.Index
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
// newIndex returns a new instance of Index.
func newIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
@ -41,7 +41,7 @@ func NewIndex() *Index {
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := NewIndex()
index := newIndex()
if err := index.Open(); err != nil {
panic(err)
}

14
test/logger.go
View file

@ -20,20 +20,20 @@ import (
"io/ioutil"
)
// BufferLogger represents a test Logger that holds log messages
// bufferLogger represents a test Logger that holds log messages
// in a buffer for review.
type BufferLogger struct {
type bufferLogger struct {
buf *bytes.Buffer
}
// NewBufferLogger returns a new instance of BufferLogger.
func NewBufferLogger() *BufferLogger {
return &BufferLogger{
func NewBufferLogger() *bufferLogger {
return &bufferLogger{
buf: &bytes.Buffer{},
}
}
func (b *BufferLogger) Printf(format string, v ...interface{}) {
func (b *bufferLogger) Printf(format string, v ...interface{}) {
s := fmt.Sprintf(format, v...)
_, err := b.buf.WriteString(s)
if err != nil {
@ -41,8 +41,8 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) {
}
}
func (b *BufferLogger) Debugf(format string, v ...interface{}) {}
func (b *bufferLogger) Debugf(format string, v ...interface{}) {}
func (b *BufferLogger) ReadAll() ([]byte, error) {
func (b *bufferLogger) ReadAll() ([]byte, error) {
return ioutil.ReadAll(b.buf)
}

20
test/pilosa.go
View file

@ -36,9 +36,9 @@ type Command struct {
commandOptions []server.CommandOption
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
stdin bytes.Buffer
stdout bytes.Buffer
stderr bytes.Buffer
}
func OptAllowedOrigins(origins []string) server.CommandOption {
@ -48,8 +48,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
}
}
// NewCommand returns a new instance of Main with a temporary data directory and random port.
func NewCommand(opts ...server.CommandOption) *Command {
// newCommand returns a new instance of Main with a temporary data directory and random port.
func newCommand(opts ...server.CommandOption) *Command {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
@ -59,9 +59,9 @@ func NewCommand(opts ...server.CommandOption) *Command {
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
m.Command.Stdin = &m.stdin
m.Command.Stdout = &m.stdout
m.Command.Stderr = &m.stderr
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
@ -73,7 +73,7 @@ func NewCommand(opts ...server.CommandOption) *Command {
// NewCommandNode returns a new instance of Command with clustering enabled.
func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
m := NewCommand(opts...)
m := newCommand(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
@ -81,7 +81,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
// MustRunCommand returns a new, running Main. Panic on error.
func MustRunCommand() *Command {
m := NewCommand()
m := newCommand()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Start(); err != nil {
panic(err)

54
translate.go
View file

@ -67,13 +67,13 @@ type TranslateFile struct {
rows map[frameKey]*index
Path string
MapSize int
mapSize int
// If non-nil, data is streamed from a primary and this is a read-only store.
PrimaryTranslateStore TranslateStore
// Delay after attempting to connect to a primary that the store will retry.
ReplicationRetryInterval time.Duration
replicationRetryInterval time.Duration
}
// NewTranslateFile returns a new instance of TranslateFile.
@ -84,9 +84,9 @@ func NewTranslateFile() *TranslateFile {
cols: make(map[string]*index),
rows: make(map[frameKey]*index),
MapSize: DefaultMapSize,
mapSize: defaultMapSize,
ReplicationRetryInterval: defaultReplicationRetryInterval,
replicationRetryInterval: defaultReplicationRetryInterval,
}
}
@ -100,7 +100,7 @@ func (s *TranslateFile) Open() (err error) {
s.w = bufio.NewWriter(s.file)
// Memory map data file.
if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
return err
}
@ -142,16 +142,16 @@ func (s *TranslateFile) Closing() <-chan struct{} {
return s.closing
}
// Size returns the number of bytes in use in the data file.
func (s *TranslateFile) Size() int64 {
// size returns the number of bytes in use in the data file.
func (s *TranslateFile) size() int64 {
s.mu.RLock()
n := s.n
s.mu.RUnlock()
return n
}
// IsReadOnly returns true if this store is being replicated from a primary store.
func (s *TranslateFile) IsReadOnly() bool {
// isReadOnly returns true if this store is being replicated from a primary store.
func (s *TranslateFile) isReadOnly() bool {
return s.PrimaryTranslateStore != nil
}
@ -193,7 +193,7 @@ func (s *TranslateFile) appendEntry(entry *LogEntry) error {
func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error {
// Move offset to the start of the id/key pairs.
offset += entry.HeaderSize()
offset += entry.headerSize()
var idx *index
switch entry.Type {
@ -270,14 +270,14 @@ func (s *TranslateFile) monitorReplication() {
select {
case <-s.closing:
return
case <-time.After(s.ReplicationRetryInterval):
case <-time.After(s.replicationRetryInterval):
log.Printf("pilosa: reconnecting to primary replica")
}
}
}
func (s *TranslateFile) replicate(ctx context.Context) error {
off := s.Size()
off := s.size()
// Connect to remote primary.
log.Printf("pilosa: replicating from offset %d", off)
@ -351,7 +351,7 @@ func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string)
s.mu.RUnlock()
// Return error if not all values could be translated and this store is read-only.
if s.IsReadOnly() {
if s.isReadOnly() {
return ret, ErrTranslateStoreReadOnly
}
@ -457,7 +457,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri
s.mu.RUnlock()
// Return error if not all values could be translated and this store is read-only.
if s.IsReadOnly() {
if s.isReadOnly() {
return ret, ErrTranslateStoreReadOnly
}
@ -538,7 +538,7 @@ func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (st
// Reader returns a reader that streams the underlying data file.
func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
rc := NewTranslateFileReader(ctx, s, offset)
rc := newTranslateFileReader(ctx, s, offset)
if err := rc.Open(); err != nil {
return nil, err
}
@ -558,8 +558,8 @@ type LogEntry struct {
Length uint64
}
// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count.
func (e *LogEntry) HeaderSize() int64 {
// headerSize returns the number of bytes required for size, type, index, frame, & pair count.
func (e *LogEntry) headerSize() int64 {
sz := uVarintSize(e.Length) + // total entry length
1 + // type
uVarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data
@ -898,8 +898,8 @@ func pow2(v uint64) uint64 {
panic("unreachable")
}
// TranslateFileReader implements a reader that continuously streams data from a store.
type TranslateFileReader struct {
// translateFileReader implements a reader that continuously streams data from a store.
type translateFileReader struct {
ctx context.Context
store *TranslateFile
file *os.File
@ -910,9 +910,9 @@ type TranslateFileReader struct {
closing chan struct{}
}
// NewTranslateFileReader returns a new instance of TranslateFileReader.
func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader {
return &TranslateFileReader{
// newTranslateFileReader returns a new instance of TranslateFileReader.
func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *translateFileReader {
return &translateFileReader{
ctx: ctx,
store: store,
offset: offset,
@ -922,7 +922,7 @@ func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset in
}
// Open initializes the reader.
func (r *TranslateFileReader) Open() (err error) {
func (r *translateFileReader) Open() (err error) {
if r.file, err = os.Open(r.store.Path); err != nil {
return err
}
@ -930,7 +930,7 @@ func (r *TranslateFileReader) Open() (err error) {
}
// Close closes the underlying file reader.
func (r *TranslateFileReader) Close() error {
func (r *translateFileReader) Close() error {
r.once.Do(func() { close(r.closing) })
if r.file != nil {
@ -941,7 +941,7 @@ func (r *TranslateFileReader) Close() error {
// Read reads the next section of the available data to p. This should always
// read from the start of an entry and read n bytes to the end of another entry.
func (r *TranslateFileReader) Read(p []byte) (n int, err error) {
func (r *translateFileReader) Read(p []byte) (n int, err error) {
for {
// Obtain notification channel before we check for new data.
notify := r.store.WriteNotify()
@ -966,8 +966,8 @@ func (r *TranslateFileReader) Read(p []byte) (n int, err error) {
}
// read writes the bytes for zero or more valid entries to p.
func (r *TranslateFileReader) read(p []byte) (n int, err error) {
sz := r.store.Size()
func (r *translateFileReader) read(p []byte) (n int, err error) {
sz := r.store.size()
// Exit if there is no new data.
if sz < r.offset {

4
translate_mapsize_all64bitsystems.go
View file

@ -2,7 +2,7 @@
package pilosa
// DefaultMapSize is the default size of mapped memory for the translate store.
// defaultMapSize is the default size of mapped memory for the translate store.
// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on
// 64bit systems.
const DefaultMapSize = 10 * (1 << 30) // 10GB
const defaultMapSize = 10 * (1 << 30) // 10GB

110
uri.go
View file

@ -21,7 +21,6 @@ import (
"strconv"
"strings"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -43,17 +42,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a-
// localhost
// :10101
type URI struct {
scheme string `json:"scheme"`
host string `json:"host"`
port uint16 `json:"port"`
Scheme string `json:"scheme"`
Host string `json:"host"`
Port uint16 `json:"port"`
}
// DefaultURI creates and returns the default URI.
func DefaultURI() *URI {
// defaultURI creates and returns the default URI.
func defaultURI() *URI {
return &URI{
scheme: "http",
host: "localhost",
port: 10101,
Scheme: "http",
Host: "localhost",
Port: 10101,
}
}
@ -69,8 +68,8 @@ func (u URIs) HostPortStrings() []string {
// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
uri := DefaultURI()
err := uri.SetHost(host)
uri := defaultURI()
err := uri.setHost(host)
if err != nil {
return nil, errors.Wrap(err, "setting uri host")
}
@ -83,44 +82,29 @@ func NewURIFromAddress(address string) (*URI, error) {
return parseAddress(address)
}
// Scheme returns the scheme of this URI.
func (u *URI) Scheme() string {
return u.scheme
}
// SetScheme sets the scheme of this URI.
func (u *URI) SetScheme(scheme string) error {
// setScheme sets the scheme of this URI.
func (u *URI) setScheme(scheme string) error {
m := schemeRegexp.FindStringSubmatch(scheme)
if m == nil {
return errors.New("invalid scheme")
}
u.scheme = scheme
u.Scheme = scheme
return nil
}
// Host returns the host of this URI.
func (u *URI) Host() string {
return u.host
}
// SetHost sets the host of this URI.
func (u *URI) SetHost(host string) error {
// setHost sets the host of this URI.
func (u *URI) setHost(host string) error {
m := hostRegexp.FindStringSubmatch(host)
if m == nil {
return errors.New("invalid host")
}
u.host = host
u.Host = host
return nil
}
// Port returns the port of this URI.
func (u *URI) Port() uint16 {
return u.port
}
// SetPort sets the port of this URI.
func (u *URI) SetPort(port uint16) {
u.port = port
u.Port = port
}
// HostPort returns `Host:Port`
@ -129,28 +113,28 @@ func (u *URI) HostPort() string {
if u == nil {
return ""
}
s := fmt.Sprintf("%s:%d", u.host, u.port)
s := fmt.Sprintf("%s:%d", u.Host, u.Port)
return s
}
// Normalize returns the address in a form usable by a HTTP client.
func (u *URI) Normalize() string {
scheme := u.scheme
// normalize returns the address in a form usable by a HTTP client.
func (u *URI) normalize() string {
scheme := u.Scheme
index := strings.Index(scheme, "+")
if index >= 0 {
scheme = scheme[:index]
}
return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port)
return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port)
}
// String returns the address as a string.
func (u URI) String() string {
return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port)
return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port)
}
// Path returns URI with path
func (u *URI) Path(path string) string {
return fmt.Sprintf("%s%s", u.Normalize(), path)
return fmt.Sprintf("%s%s", u.normalize(), path)
}
// The following methods are required to implement pflag Value interface.
@ -191,41 +175,13 @@ func parseAddress(address string) (uri *URI, err error) {
}
}
uri = &URI{
scheme: scheme,
host: host,
port: uint16(port),
Scheme: scheme,
Host: host,
Port: uint16(port),
}
return uri, nil
}
// Encode converts o into its internal representation.
func (u URI) Encode() *internal.URI {
return encodeURI(u)
}
func encodeURI(u URI) *internal.URI {
return &internal.URI{
Scheme: u.scheme,
Host: u.host,
Port: uint32(u.port),
}
}
func DecodeURI(i *internal.URI) URI {
return decodeURI(i)
}
func decodeURI(i *internal.URI) URI {
if i == nil {
return URI{}
}
return URI{
scheme: i.Scheme,
host: i.Host,
port: uint16(i.Port),
}
}
// MarshalJSON marshals URI into a JSON-encoded byte slice.
func (u *URI) MarshalJSON() ([]byte, error) {
var output struct {
@ -233,9 +189,9 @@ func (u *URI) MarshalJSON() ([]byte, error) {
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
output.Scheme = u.scheme
output.Host = u.host
output.Port = u.port
output.Scheme = u.Scheme
output.Host = u.Host
output.Port = u.Port
return json.Marshal(output)
}
@ -249,8 +205,8 @@ func (u *URI) UnmarshalJSON(b []byte) error {
if err := json.Unmarshal(b, &input); err != nil {
return err
}
u.scheme = input.Scheme
u.host = input.Host
u.port = input.Port
u.Scheme = input.Scheme
u.Host = input.Host
u.Port = input.Port
return nil
}

46
uri_internal_test.go
View file

@ -17,7 +17,7 @@ package pilosa
import "testing"
func TestDefaultURI(t *testing.T) {
uri := DefaultURI()
uri := defaultURI()
compare(t, uri, "http", "localhost", 10101)
}
@ -60,7 +60,7 @@ func TestNormalizedAddress(t *testing.T) {
if err != nil {
t.Fatalf("Can't parse address")
}
if uri.Normalize() != "http://big-data.pilosa.com:6888" {
if uri.normalize() != "http://big-data.pilosa.com:6888" {
t.Fatalf("Normalized address is not normal")
}
}
@ -77,49 +77,49 @@ func TestURIPath(t *testing.T) {
}
func TestSetScheme(t *testing.T) {
uri := DefaultURI()
uri := defaultURI()
target := "fun"
err := uri.SetScheme(target)
err := uri.setScheme(target)
if err != nil {
t.Fatal(err)
}
if uri.Scheme() != target {
t.Fatalf("%s != %s", uri.Scheme(), target)
if uri.Scheme != target {
t.Fatalf("%s != %s", uri.Scheme, target)
}
}
func TestSetHost(t *testing.T) {
uri := DefaultURI()
uri := defaultURI()
target := "10.20.30.40"
err := uri.SetHost(target)
err := uri.setHost(target)
if err != nil {
t.Fatal(err)
}
if uri.Host() != target {
t.Fatalf("%s != %s", uri.host, target)
if uri.Host != target {
t.Fatalf("%s != %s", uri.Host, target)
}
}
func TestSetPort(t *testing.T) {
uri := DefaultURI()
uri := defaultURI()
target := uint16(9999)
uri.SetPort(target)
if uri.Port() != target {
t.Fatalf("%d != %d", uri.port, target)
if uri.Port != target {
t.Fatalf("%d != %d", uri.Port, target)
}
}
func TestSetInvalidScheme(t *testing.T) {
uri := DefaultURI()
err := uri.SetScheme("?invalid")
uri := defaultURI()
err := uri.setScheme("?invalid")
if err == nil {
t.Fatalf("Should have failed")
}
}
func TestSetInvalidHost(t *testing.T) {
uri := DefaultURI()
err := uri.SetHost("index?.pilosa.com")
uri := defaultURI()
err := uri.setHost("index?.pilosa.com")
if err == nil {
t.Fatalf("Should have failed")
}
@ -137,14 +137,14 @@ func TestHostPort(t *testing.T) {
}
func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) {
if uri.Scheme() != scheme {
t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme)
if uri.Scheme != scheme {
t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme)
}
if uri.Host() != host {
t.Fatalf("Host does not match: %s != %s", uri.host, host)
if uri.Host != host {
t.Fatalf("Host does not match: %s != %s", uri.Host, host)
}
if uri.Port() != port {
t.Fatalf("Port does not match: %d != %d", uri.port, port)
if uri.Port != port {
t.Fatalf("Port does not match: %d != %d", uri.Port, port)
}
}

48
utils_internal_test.go
View file

@ -24,7 +24,6 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -38,7 +37,7 @@ func NewTestCluster(n int) *cluster {
c.ReplicaN = 1
c.Hasher = NewTestModHasher()
c.Path = path
c.Topology = NewTopology()
c.Topology = newTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &Node{
@ -56,16 +55,16 @@ func NewTestCluster(n int) *cluster {
// NewTestURI is a test URI creator that intentionally swallows errors.
func NewTestURI(scheme, host string, port uint16) URI {
uri := DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri := defaultURI()
uri.setScheme(scheme)
uri.setHost(host)
uri.SetPort(port)
return *uri
}
func NewTestURIFromHostPort(host string, port uint16) URI {
uri := DefaultURI()
uri.SetHost(host)
uri := defaultURI()
uri.setHost(host)
uri.SetPort(port)
return *uri
}
@ -162,7 +161,7 @@ func (t *ClusterCluster) addNode() error {
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &nodeEvent{
ev := &NodeEvent{
Event: NodeJoin,
Node: c.Node,
}
@ -186,7 +185,7 @@ func (t *ClusterCluster) addNode() error {
// WriteTopology writes the given topology to disk.
func (t *ClusterCluster) WriteTopology(path string, top *Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
if buf, err := proto.Marshal(top.encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
@ -226,7 +225,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
c.ReplicaN = 1
c.Hasher = NewTestModHasher()
c.Path = path
c.Topology = NewTopology()
c.Topology = newTopology()
c.holder = h
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
@ -278,7 +277,7 @@ func (t *ClusterCluster) Open() error {
if err := c.holder.Open(); err != nil {
return err
}
if err := c.setNodeState(NodeStateReady); err != nil {
if err := c.setNodeState(nodeStateReady); err != nil {
return err
}
}
@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error {
}
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *ClusterCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
func (t *ClusterCluster) SendSync(m Message) error {
switch obj := m.(type) {
case *ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.mergeClusterStatus(obj)
@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error {
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *ClusterCluster) SendAsync(pb proto.Message) error {
func (t *ClusterCluster) SendAsync(Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
func (t *ClusterCluster) SendTo(to *Node, m Message) error {
switch obj := m.(type) {
case *ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
case *ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.markResizeInstructionComplete(obj)
}
@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI)
instrNode := DecodeNode(instr.Node)
instrNode := instr.Node
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
@ -365,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
}
for _, src := range instr.Sources {
srcNode := DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcCluster := t.clusterByID(src.Node.ID)
srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
@ -405,6 +403,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
complete.Error = err.Error()
}
node := DecodeNode(instr.Coordinator)
node := instr.Coordinator
return t.SendTo(node, complete)
}

9
view.go
View file

@ -22,7 +22,6 @@ import (
"strings"
"sync"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) {
// Send the create shard message to all nodes.
err := v.broadcaster.SendSync(
&internal.CreateShardMessage{
&CreateShardMessage{
Index: v.index,
Shard: shard,
})
@ -422,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*
return r, nil
}
// viewInfo represents schema information for a view.
type viewInfo struct {
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`
}
type viewInfoSlice []*viewInfo
type viewInfoSlice []*ViewInfo
func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int { return len(p) }