scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 03:17:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into docs-updates-1.0

Browse source
This commit is contained in:
Travis Turner 2018-07-05 21:53:37 -05:00 • committed by GitHub
commit cda0edd484
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
52 changed files with 2109 additions and 1577 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

147
api.go
View file

@ -26,8 +26,6 @@ import (
"strings"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -38,6 +36,8 @@ type API struct {
holder *Holder
cluster *cluster
server *Server
Serializer Serializer
}
// APIOption is a functional option type for pilosa.API
@ -48,6 +48,7 @@ func OptAPIServer(s *Server) APIOption {
a.server = s
a.holder = s.holder
a.cluster = s.cluster
a.Serializer = s.serializer
return nil
}
}
@ -149,6 +150,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
return resp, nil
}
func (api *API) Holder() *Holder {
return api.server.Holder()
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
@ -185,12 +190,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
}
// Send the create index message to all nodes.
err = api.server.SendSync(
&internal.CreateIndexMessage{
&CreateIndexMessage{
Index: indexName,
Meta: options.Encode(),
Meta: &options,
})
if err != nil {
api.server.logger.Printf("problem sending CreateIndex message: %s", err)
return nil, errors.Wrap(err, "sending CreateIndex message")
}
api.holder.Stats.Count("createIndex", 1, 1.0)
@ -224,7 +228,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
}
// Send the delete index message to all nodes.
err = api.server.SendSync(
&internal.DeleteIndexMessage{
&DeleteIndexMessage{
Index: indexName,
})
if err != nil {
@ -238,16 +242,18 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
// CreateField makes the named field in the named index with the given options.
// This method currently only takes a single functional option, but that may be
// changed in the future to support multiple options.
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts FieldOption) (*Field, error) {
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
if err := api.validate(apiCreateField); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Apply functional option.
// Apply functional options.
fo := FieldOptions{}
err := opts(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
for _, opt := range opts {
err := opt(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
}
// Find index.
@ -257,17 +263,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
}
// Create field.
field, err := index.CreateField(fieldName, fo)
field, err := index.CreateField(fieldName, opts...)
if err != nil {
return nil, errors.Wrap(err, "creating field")
}
// Send the create field message to all nodes.
err = api.server.SendSync(
&internal.CreateFieldMessage{
&CreateFieldMessage{
Index: indexName,
Field: fieldName,
Meta: fo.Encode(),
Meta: &fo,
})
if err != nil {
api.server.logger.Printf("problem sending CreateField message: %s", err)
@ -311,7 +317,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
// Send the delete field message to all nodes.
err := api.server.SendSync(
&internal.DeleteFieldMessage{
&DeleteFieldMessage{
Index: indexName,
Field: fieldName,
})
@ -331,8 +337,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Validate that this handler owns the shard.
if !api.cluster.ownsShard(api.LocalID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
return ErrClusterDoesNotOwnShard
}
@ -370,55 +376,6 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64)
return api.cluster.shardNodes(indexName, shard), nil
}
// MarshalFragment returns an object which can write the specified fragment's data
// to an io.Writer. The serialized data can be read back into a fragment with
// the UnmarshalFragment API call.
func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) {
if err := api.validate(apiMarshalFragment); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.holder.fragment(indexName, fieldName, viewStandard, shard)
if f == nil {
return nil, ErrFragmentNotFound
}
return f, nil
}
// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
// Reader which was previously written by MarshalFragment to populate the
// fragment's data.
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error {
if err := api.validate(apiUnmarshalFragment); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve field.
f := api.holder.Field(indexName, fieldName)
if f == nil {
return ErrFieldNotFound
}
// Retrieve view.
view, err := f.createViewIfNotExists(viewStandard)
if err != nil {
return errors.Wrap(err, "creating view")
}
// Retrieve fragment from field.
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
// Read fragment in from request body.
if _, err := frag.ReadFrom(reader); err != nil {
return errors.Wrap(err, "reading fragment")
}
return nil
}
// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to
// return anything useful. Currently it returns protobuf encoded row and column
// ids from a "block" which is a subdivision of a fragment.
@ -431,8 +388,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read body error"))
}
var req internal.BlockDataRequest
if err := proto.Unmarshal(reqBytes, &req); err != nil {
var req BlockDataRequest
if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error"))
}
@ -442,11 +399,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
return nil, ErrFragmentNotFound
}
var resp = internal.BlockDataResponse{}
var resp = BlockDataResponse{}
resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block))
// Encode response.
buf, err := proto.Marshal(&resp)
buf, err := api.Serializer.Marshal(&resp)
if err != nil {
return nil, errors.Wrap(err, "merge block response encoding error")
}
@ -477,13 +434,19 @@ func (api *API) Hosts(ctx context.Context) []*Node {
return api.cluster.Nodes
}
// Node gets the ID, URI and coordinator status for this particular node.
func (api *API) Node() *Node {
node := api.server.node()
return &node
}
// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
func (api *API) RecalculateCaches(ctx context.Context) error {
if err := api.validate(apiRecalculateCaches); err != nil {
return errors.Wrap(err, "validating api method")
}
err := api.server.SendSync(&internal.RecalculateCaches{})
err := api.server.SendSync(&RecalculateCaches{})
if err != nil {
return errors.Wrap(err, "broacasting message")
}
@ -504,28 +467,24 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
return errors.Wrap(err, "reading body")
}
// Marshal into request object.
pb, err := UnmarshalMessage(body)
typ := body[0]
msg := getMessage(typ)
err = api.server.serializer.Unmarshal(body[1:], msg)
if err != nil {
return errors.Wrap(err, "unmarshaling message")
return errors.Wrap(err, "deserializing cluster message")
}
// Forward the error message.
if err := api.server.ReceiveMessage(pb); err != nil {
if err := api.server.receiveMessage(msg); err != nil {
return errors.Wrap(err, "receiving message")
}
return nil
}
// LocalID returns the current node's ID.
func (api *API) LocalID() string {
return api.cluster.Node.ID
}
// Schema returns information about each index in Pilosa including which fields
// and views they contain.
// they contain.
func (api *API) Schema(ctx context.Context) []*IndexInfo {
return api.holder.Schema()
return api.holder.limitedSchema()
}
// Views returns the views in the given field.
@ -567,7 +526,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
// Send the delete view message to all nodes.
err := api.server.SendSync(
&internal.DeleteViewMessage{
&DeleteViewMessage{
Index: indexName,
Field: fieldName,
View: viewName,
@ -649,7 +608,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
}
// Import bulk imports data into a particular index,field,shard.
func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
func (api *API) Import(ctx context.Context, req *ImportRequest) error {
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -678,7 +637,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
}
// ImportValue bulk imports values into a particular field.
func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error {
func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error {
if err := api.validate(apiImportValue); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -720,8 +679,8 @@ func (api *API) LongQueryTime() time.Duration {
func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) {
// Validate that this handler owns the shard.
if !api.cluster.ownsShard(api.LocalID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
return nil, nil, ErrClusterDoesNotOwnShard
}
@ -755,15 +714,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
}
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == api.LocalID() {
if newNode.ID == api.Node().ID {
return oldNode, newNode, api.cluster.setCoordinator(newNode)
}
// Send the set-coordinator message to new node.
err = api.server.SendTo(
newNode,
&internal.SetCoordinatorMessage{
New: EncodeNode(newNode),
&SetCoordinatorMessage{
New: newNode,
})
if err != nil {
return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err)
@ -888,7 +847,6 @@ const (
apiIndexAttrDiff
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
//apiMaxShards // not implemented
apiQuery
apiRecalculateCaches
@ -899,15 +857,13 @@ const (
apiShardNodes
//apiState // not implemented
//apiStatsWithTags // not implemented
apiUnmarshalFragment
//apiVersion // not implemented
apiViews
)
var methodsCommon = map[apiMethod]struct{}{
apiClusterMessage: struct{}{},
apiMarshalFragment: struct{}{},
apiSetCoordinator: struct{}{},
apiClusterMessage: struct{}{},
apiSetCoordinator: struct{}{},
}
var methodsResizing = map[apiMethod]struct{}{
@ -933,6 +889,5 @@ var methodsNormal = map[apiMethod]struct{}{
apiRecalculateCaches: struct{}{},
apiRemoveNode: struct{}{},
apiShardNodes: struct{}{},
apiUnmarshalFragment: struct{}{},
apiViews: struct{}{},
}

4
apimethod_string.go
View file

@ -4,9 +4,9 @@ package pilosa
import "strconv"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 224, 232, 252, 265, 279, 296, 309, 329, 337}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

155
broadcast.go
View file

@ -16,20 +16,27 @@ package pilosa
import (
"fmt"
"reflect"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
// Serializer is an interface for serializing pilosa types to bytes and back.
type Serializer interface {
Marshal(Message) ([]byte, error)
Unmarshal([]byte, Message) error
}
// broadcaster is an interface for broadcasting messages.
type broadcaster interface {
SendSync(pb proto.Message) error
SendAsync(pb proto.Message) error
SendTo(to *Node, pb proto.Message) error
SendSync(Message) error
SendAsync(Message) error
SendTo(*Node, Message) error
}
// Message is the interface implemented by all core pilosa types which can be serialized to messages.
// TODO add at least a single "isMessage()" method.
type Message interface{}
func init() {
NopBroadcaster = &nopBroadcaster{}
}
@ -40,13 +47,13 @@ var NopBroadcaster broadcaster
type nopBroadcaster struct{}
// SendSync A no-op implementation of Broadcaster SendSync method.
func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil }
func (nopBroadcaster) SendSync(Message) error { return nil }
// SendAsync A no-op implementation of Broadcaster SendAsync method.
func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil }
func (nopBroadcaster) SendAsync(Message) error { return nil }
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil }
func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
// Broadcast message types.
const (
@ -65,94 +72,94 @@ const (
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeNodeEvent
messageTypeNodeStatus
)
// MarshalMessage encodes the protobuf message into a byte slice.
func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateShardMessage:
typ = messageTypeCreateShard
case *internal.CreateIndexMessage:
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = messageTypeDeleteIndex
case *internal.CreateFieldMessage:
typ = messageTypeCreateField
case *internal.DeleteFieldMessage:
typ = messageTypeDeleteField
case *internal.CreateViewMessage:
typ = messageTypeCreateView
case *internal.DeleteViewMessage:
typ = messageTypeDeleteView
case *internal.ClusterStatus:
typ = messageTypeClusterStatus
case *internal.ResizeInstruction:
typ = messageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = messageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = messageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = messageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = messageTypeNodeState
case *internal.RecalculateCaches:
typ = messageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = messageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
buf, err := proto.Marshal(m)
// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
// type info which is used by the internal messaging stuff.
func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
typ := getMessageType(m)
buf, err := s.Marshal(m)
if err != nil {
return nil, errors.Wrap(err, "marshalling")
return nil, errors.Wrap(err, "marshaling")
}
return append([]byte{typ}, buf...), nil
}
// UnmarshalMessage decodes the byte slice into a protobuf message.
func UnmarshalMessage(buf []byte) (proto.Message, error) {
typ, buf := buf[0], buf[1:]
var m proto.Message
func getMessage(typ byte) Message {
switch typ {
case messageTypeCreateShard:
m = &internal.CreateShardMessage{}
return &CreateShardMessage{}
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
return &CreateIndexMessage{}
case messageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
return &DeleteIndexMessage{}
case messageTypeCreateField:
m = &internal.CreateFieldMessage{}
return &CreateFieldMessage{}
case messageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
return &DeleteFieldMessage{}
case messageTypeCreateView:
m = &internal.CreateViewMessage{}
return &CreateViewMessage{}
case messageTypeDeleteView:
m = &internal.DeleteViewMessage{}
return &DeleteViewMessage{}
case messageTypeClusterStatus:
m = &internal.ClusterStatus{}
return &ClusterStatus{}
case messageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
return &ResizeInstruction{}
case messageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
return &ResizeInstructionComplete{}
case messageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
return &SetCoordinatorMessage{}
case messageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
return &UpdateCoordinatorMessage{}
case messageTypeNodeState:
m = &internal.NodeStateMessage{}
return &NodeStateMessage{}
case messageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
return &RecalculateCaches{}
case messageTypeNodeEvent:
m = &internal.NodeEventMessage{}
return &NodeEvent{}
case messageTypeNodeStatus:
return &NodeStatus{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
panic(fmt.Sprintf("unknown message type %d", typ))
}
}
func getMessageType(m Message) byte {
switch m.(type) {
case *CreateShardMessage:
return messageTypeCreateShard
case *CreateIndexMessage:
return messageTypeCreateIndex
case *DeleteIndexMessage:
return messageTypeDeleteIndex
case *CreateFieldMessage:
return messageTypeCreateField
case *DeleteFieldMessage:
return messageTypeDeleteField
case *CreateViewMessage:
return messageTypeCreateView
case *DeleteViewMessage:
return messageTypeDeleteView
case *ClusterStatus:
return messageTypeClusterStatus
case *ResizeInstruction:
return messageTypeResizeInstruction
case *ResizeInstructionComplete:
return messageTypeResizeInstructionComplete
case *SetCoordinatorMessage:
return messageTypeSetCoordinator
case *UpdateCoordinatorMessage:
return messageTypeUpdateCoordinator
case *NodeStateMessage:
return messageTypeNodeState
case *RecalculateCaches:
return messageTypeRecalculateCaches
case *NodeEvent:
return messageTypeNodeEvent
case *NodeStatus:
return messageTypeNodeStatus
default:
panic(fmt.Sprintf("don't have type for message %#v", m))
}
if err := proto.Unmarshal(buf, m); err != nil {
return nil, errors.Wrap(err, "unmarshalling")
}
return m, nil
}

51
broadcast_test.go
View file

@ -1,51 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"reflect"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// Ensure a message can be marshaled and unmarshaled.
func TestMessage_Marshal(t *testing.T) {
testMessageMarshal(t, &internal.CreateShardMessage{
Index: "i",
Shard: 8,
})
testMessageMarshal(t, &internal.DeleteIndexMessage{
Index: "i",
})
}
func testMessageMarshal(t *testing.T, m proto.Message) {
marshalled, err := pilosa.MarshalMessage(m)
if err != nil {
t.Fatal(err)
}
unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(unmarshalled, m) {
t.Fatalf("unexpected message marshalling: %s", unmarshalled)
}
}

33
cache.go
View file

@ -22,7 +22,6 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/lru"
)
@ -318,22 +317,6 @@ type Pair struct {
Count uint64 `json:"count"`
}
func encodePair(p Pair) *internal.Pair {
return &internal.Pair{
ID: p.ID,
Key: p.Key,
Count: p.Count,
}
}
func decodePair(pb *internal.Pair) Pair {
return Pair{
ID: pb.ID,
Key: pb.Key,
Count: pb.Count,
}
}
// Pairs is a sortable slice of Pair objects.
type Pairs []Pair
@ -409,22 +392,6 @@ func (p Pairs) String() string {
return buf.String()
}
func EncodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])
}
return other
}
func decodePairs(a []*internal.Pair) []Pair {
other := make([]Pair, len(a))
for i := range a {
other[i] = decodePair(a[i])
}
return other
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

19
client.go
View file

@ -3,9 +3,6 @@ package pilosa
import (
"context"
"io"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// Bit represents the intersection of a row and a column. It can be specifed by
@ -36,8 +33,8 @@ type InternalClient interface {
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
@ -49,19 +46,19 @@ type InternalClient interface {
BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
}
//===============
type InternalQueryClient interface {
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
}
type NopInternalQueryClient struct{}
func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
@ -91,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In
func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
return nil, nil
}
func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {
@ -128,7 +125,7 @@ func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index s
func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error {
func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
return nil
}
func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {

256
cluster.go
View file

@ -45,8 +45,7 @@ const (
ClusterStateResizing = "RESIZING"
// NodeState represents the state of a node during startup.
NodeStateLoading = "LOADING"
NodeStateReady = "READY"
NodeStateReady = "READY"
// resizeJob states.
resizeJobStateRunning = "RUNNING"
@ -69,52 +68,6 @@ func (n Node) String() string {
return fmt.Sprintf("Node: %s", n.ID)
}
// EncodeNodes converts a slice of Nodes into its internal representation.
func EncodeNodes(a []*Node) []*internal.Node {
other := make([]*internal.Node, len(a))
for i := range a {
other[i] = EncodeNode(a[i])
}
return other
}
// EncodeNode converts a Node into its internal representation.
func EncodeNode(n *Node) *internal.Node {
return &internal.Node{
ID: n.ID,
URI: n.URI.Encode(),
IsCoordinator: n.IsCoordinator,
}
}
// DecodeNodes converts a proto message into a slice of Nodes.
func DecodeNodes(a []*internal.Node) []*Node {
if len(a) == 0 {
return nil
}
other := make([]*Node, len(a))
for i := range a {
other[i] = DecodeNode(a[i])
}
return other
}
// DecodeNode converts a proto message into a Node.
func DecodeNode(node *internal.Node) *Node {
return &Node{
ID: node.ID,
URI: decodeURI(node.URI),
IsCoordinator: node.IsCoordinator,
}
}
func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent {
return &nodeEvent{
Event: NodeEventType(ne.Event),
Node: DecodeNode(ne.Node),
}
}
// Nodes represents a list of nodes.
type Nodes []*Node
@ -314,8 +267,8 @@ func (c *cluster) setCoordinator(n *Node) error {
c.mu.Unlock()
// Send the update coordinator message to all nodes.
err := c.broadcaster.SendSync(
&internal.UpdateCoordinatorMessage{
New: EncodeNode(n),
&UpdateCoordinatorMessage{
New: n,
})
if err != nil {
return fmt.Errorf("problem sending UpdateCoordinator message: %v", err)
@ -469,7 +422,7 @@ func (c *cluster) setNodeState(state string) error {
}
// Send node state to coordinator.
ns := &internal.NodeStateMessage{
ns := &NodeStateMessage{
NodeID: c.Node.ID,
State: state,
}
@ -506,12 +459,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
return nil
}
// Status returns the internal ClusterStatus representation.
func (c *cluster) Status() *internal.ClusterStatus {
return &internal.ClusterStatus{
// Status returns the the cluster's status including what nodes it contains, its ID, and current state.
func (c *cluster) Status() *ClusterStatus {
return &ClusterStatus{
ClusterID: c.id,
State: c.state,
Nodes: EncodeNodes(c.Nodes),
Nodes: c.Nodes,
}
}
@ -686,8 +639,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error)
// fragSources returns a list of ResizeSources - for each node in the `to` cluster -
// required to move from cluster `c` to cluster `to`.
func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) {
m := make(map[string][]*internal.ResizeSource)
func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) {
m := make(map[string][]*ResizeSource)
// Determine if a node is being added or removed.
action, diffNodeID, err := c.diff(to)
@ -746,7 +699,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
// Get the ResizeSource for each diff.
for nodeID, diff := range diffs {
m[nodeID] = []*internal.ResizeSource{}
m[nodeID] = []*ResizeSource{}
for _, frag := range diff {
// If there is no valid source node ID for a fragment,
// it likely means that the replica factor was not
@ -757,8 +710,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R
return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)")
}
src := &internal.ResizeSource{
Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)),
src := &ResizeSource{
Node: c.unprotectedNodeByID(srcNodeID),
Index: idx.Name(),
Field: frag.field,
View: frag.view,
@ -902,9 +855,9 @@ func (c *cluster) waitForStarted() error {
// TODO: Because the normal code path already sends a NodeJoin event (via
// memberlist), this it a bit redundant in most cases. Perhaps determine
// that the node has been restarted and don't do this step.
msg := &internal.NodeEventMessage{
Event: uint32(NodeJoin),
Node: EncodeNode(c.Node),
msg := &NodeEvent{
Event: NodeJoin,
Node: c.Node,
}
if err := c.broadcaster.SendSync(msg); err != nil {
return fmt.Errorf("sending restart NodeJoin: %v", err)
@ -1014,8 +967,8 @@ func (c *cluster) setStateAndBroadcast(state string) error {
return c.broadcaster.SendSync(c.Status())
}
func (c *cluster) sendTo(node *Node, msg proto.Message) error {
if err := c.broadcaster.SendTo(node, msg); err != nil {
func (c *cluster) sendTo(node *Node, m Message) error {
if err := c.broadcaster.SendTo(node, m); err != nil {
return errors.Wrap(err, "sending")
}
return nil
@ -1121,7 +1074,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
}
// multiIndex is a map of sources initialized with all the nodes in toCluster.
multiIndex := make(map[string][]*internal.ResizeSource)
multiIndex := make(map[string][]*ResizeSource)
for _, n := range toCluster.Nodes {
multiIndex[n.ID] = nil
@ -1145,12 +1098,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob,
j.IDs[id] = true
continue
}
instr := &internal.ResizeInstruction{
instr := &ResizeInstruction{
JobID: j.ID,
Node: EncodeNode(toCluster.unprotectedNodeByID(id)),
Coordinator: EncodeNode(c.coordinatorNode()),
Node: toCluster.unprotectedNodeByID(id),
Coordinator: c.coordinatorNode(),
Sources: sources,
Schema: c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node.
Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node.
ClusterStatus: c.Status(),
}
j.Instructions = append(j.Instructions, instr)
@ -1176,7 +1129,7 @@ func (c *cluster) completeCurrentJob(state string) error {
}
// followResizeInstruction is run by any node that receives a ResizeInstruction.
func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error {
func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
c.logger.Printf("follow resize instruction on %s", c.Node.ID)
// Make sure the cluster status on this node agrees with the Coordinator
// before attempting a resize.
@ -1194,7 +1147,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
<-c.holder.opened
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
@ -1213,7 +1166,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
for _, src := range instr.Sources {
c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
srcURI := decodeURI(src.Node.URI)
srcURI := src.Node.URI
// Retrieve field.
f := c.holder.Field(src.Index, src.Field)
@ -1265,14 +1218,14 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
complete.Error = err.Error()
}
if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil {
if err := c.sendTo(instr.Coordinator, complete); err != nil {
c.logger.Printf("sending resizeInstructionComplete error: err=%s", err)
}
}()
return nil
}
func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error {
func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error {
j := c.job(complete.JobID)
@ -1309,7 +1262,7 @@ func (c *cluster) job(id int64) *resizeJob {
type resizeJob struct {
ID int64
IDs map[string]bool
Instructions []*internal.ResizeInstruction
Instructions []*ResizeInstruction
Broadcaster broadcaster
action string
@ -1412,7 +1365,7 @@ func (j *resizeJob) distributeResizeInstructions() error {
// a dummy node object to use in the SendTo() method.
node := &Node{
ID: instr.Node.ID,
URI: decodeURI(instr.Node.URI),
URI: instr.Node.URI,
}
j.Logger.Printf("send resize instructions: %v", instr)
if err := j.Broadcaster.SendTo(node, instr); err != nil {
@ -1553,32 +1506,6 @@ func (c *cluster) saveTopology() error {
return nil
}
func encodeTopology(topology *Topology) *internal.Topology {
if topology == nil {
return nil
}
return &internal.Topology{
ClusterID: topology.ClusterID,
NodeIDs: topology.NodeIDs,
}
}
func decodeTopology(topology *internal.Topology) (*Topology, error) {
if topology == nil {
return nil, nil
}
t := NewTopology()
t.ClusterID = topology.ClusterID
t.NodeIDs = topology.NodeIDs
sort.Slice(t.NodeIDs,
func(i, j int) bool {
return t.NodeIDs[i] < t.NodeIDs[j]
})
return t, nil
}
func (c *cluster) considerTopology() error {
// Create ClusterID if one does not already exist.
if c.id == "" {
@ -1612,7 +1539,7 @@ func (c *cluster) considerTopology() error {
}
// ReceiveEvent represents an implementation of EventHandler.
func (c *cluster) ReceiveEvent(e *nodeEvent) error {
func (c *cluster) ReceiveEvent(e *NodeEvent) error {
// Ignore events sent from this node.
if e.Node.ID == c.Node.ID {
return nil
@ -1752,7 +1679,7 @@ func (c *cluster) nodeLeave(node *Node) error {
return nil
}
func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.logger.Printf("merge cluster status: %v", cs)
@ -1764,7 +1691,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error {
// Set ClusterID.
c.setID(cs.ClusterID)
officialNodes := DecodeNodes(cs.Nodes)
officialNodes := cs.Nodes
// Add all nodes from the coordinator.
for _, node := range officialNodes {
@ -1813,3 +1740,120 @@ func (c *cluster) setStatic(hosts []string) error {
}
return nil
}
type ClusterStatus struct {
ClusterID string
State string
Nodes []*Node
}
type ResizeInstruction struct {
JobID int64
Node *Node
Coordinator *Node
Sources []*ResizeSource
Schema *Schema
ClusterStatus *ClusterStatus
}
type ResizeSource struct {
Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"`
View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"`
Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
}
// Schema contains information about indexes and their configuration.
type Schema struct {
Indexes []*IndexInfo
}
func encodeTopology(topology *Topology) *internal.Topology {
if topology == nil {
return nil
}
return &internal.Topology{
ClusterID: topology.ClusterID,
NodeIDs: topology.NodeIDs,
}
}
func decodeTopology(topology *internal.Topology) (*Topology, error) {
if topology == nil {
return nil, nil
}
t := NewTopology()
t.ClusterID = topology.ClusterID
t.NodeIDs = topology.NodeIDs
sort.Slice(t.NodeIDs,
func(i, j int) bool {
return t.NodeIDs[i] < t.NodeIDs[j]
})
return t, nil
}
type CreateShardMessage struct {
Index string
Shard uint64
}
type CreateIndexMessage struct {
Index string
Meta *IndexOptions
}
type DeleteIndexMessage struct {
Index string
}
type CreateFieldMessage struct {
Index string
Field string
Meta *FieldOptions
}
type DeleteFieldMessage struct {
Index string
Field string
}
type CreateViewMessage struct {
Index string
Field string
View string
}
type DeleteViewMessage struct {
Index string
Field string
View string
}
type ResizeInstructionComplete struct {
JobID int64
Node *Node
Error string
}
type SetCoordinatorMessage struct {
New *Node
}
type UpdateCoordinatorMessage struct {
New *Node
}
type NodeStateMessage struct {
NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
}
type NodeStatus struct {
Node *Node
MaxShards map[string]uint64
Schema *Schema
}
type RecalculateCaches struct{}

45
cluster_internal_test.go
View file

@ -24,7 +24,6 @@ import (
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -150,7 +149,7 @@ func TestFragSources(t *testing.T) {
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
field, err := idx.CreateFieldIfNotExists("f", FieldOptions{})
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) {
from *cluster
to *cluster
idx *Index
expected map[string][]*internal.ResizeSource
expected map[string][]*ResizeSource
err string
}{
{
from: c1,
to: c2,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{},
"node1": []*internal.ResizeSource{},
"node2": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{},
"node1": []*ResizeSource{},
"node2": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) {
from: c4,
to: c3,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{
{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
"node1": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) {
from: c5,
to: c4,
idx: idx,
expected: map[string][]*internal.ResizeSource{
"node0": []*internal.ResizeSource{
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
expected: map[string][]*ResizeSource{
"node0": []*ResizeSource{
{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
},
"node1": []*internal.ResizeSource{
{&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
"node1": []*ResizeSource{
{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
},
"node2": []*internal.ResizeSource{},
"node2": []*ResizeSource{},
},
err: "",
},
@ -697,7 +696,7 @@ func TestCluster_ResizeStates(t *testing.T) {
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", 1, 101, nil)

2
ctl/export_test.go
View file

@ -49,7 +49,7 @@ func TestExportCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewExportCommand(stdin, stdout, stderr)
hostport := cmd.Server.URI.HostPort()
hostport := cmd.API.Node().URI.HostPort()
cm.Host = hostport
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))

8
ctl/import_test.go
View file

@ -62,7 +62,7 @@ func TestImportCommand_Run(t *testing.T) {
}
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Host = cmd.API.Node().URI.HostPort()
cm.Index = "i"
cm.Field = "f"
@ -87,7 +87,7 @@ func TestImportCommand_RunValue(t *testing.T) {
}
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Host = cmd.API.Node().URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
@ -107,7 +107,7 @@ func TestImportCommand_InvalidFile(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = cmd.Server.URI.HostPort()
cm.Host = cmd.API.Node().URI.HostPort()
cm.Index = "i"
cm.Field = "f"
file, err := ioutil.TempFile("", "import.csv")
@ -188,7 +188,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
t.Fatal(err)
}
cm.Host = cmd.Server.URI.HostPort()
cm.Host = cmd.API.Node().URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))

1037
encoding/proto/proto.go Normal file
View file

File diff suppressed because it is too large Load diff

4
event.go
View file

@ -23,8 +23,8 @@ const (
NodeUpdate
)
// nodeEvent is a single event related to node activity in the cluster.
type nodeEvent struct {
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
}

191
executor.go
View file

@ -20,7 +20,6 @@ import (
"sort"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -184,9 +183,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, shards, opt)
case "Set":
return e.executeSetBit(ctx, index, c, opt)
case "SetValue":
return nil, e.executeSetValue(ctx, index, c, opt)
return e.executeSet(ctx, index, c, opt)
case "SetRowAttrs":
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
@ -337,7 +334,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "map reduce")
}
// Attach attributes for Row() calls.
@ -1060,8 +1057,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
return ret, nil
}
// executeSetBit executes a Set() call.
func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Set() argument required: field")
@ -1077,14 +1074,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
// Read colID using labels.
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading Set() column: %v", err)
@ -1092,20 +1082,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return false, fmt.Errorf("Set() column argument '%v' required", columnLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if f.Type() == FieldTypeInt {
// Read remaining fields using labels.
rowVal, ok, err := c.IntArg(fieldName)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
} else {
// Read remaining fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
}
// executeSetBitField executes a Set() call for a specific view.
// executeSetBitField executes a Set() call for a specific field.
func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) {
shard := colID / ShardWidth
ret := false
@ -1137,64 +1147,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.
return ret, nil
}
// executeSetValue executes a SetValue() call.
func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error {
// Parse labels.
columnID, ok, err := c.UintArg(columnLabel)
if err != nil {
return fmt.Errorf("reading SetValue() column: %v", err)
} else if !ok {
return fmt.Errorf("SetValue() column field '%v' required", columnLabel)
}
// executeSetValueField executes a Set() call for a specific int field.
func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) {
shard := colID / ShardWidth
ret := false
// Copy args and remove reserved fields.
args := pql.CopyArgs(c.Args)
// While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion.
// Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility.
delete(args, columnLabel)
// Set values.
for name, value := range args {
// Retrieve field.
field := e.Holder.Field(index, name)
if field == nil {
return ErrFieldNotFound
}
switch value := value.(type) {
case int64:
if _, err := field.SetValue(columnID, value); err != nil {
return err
for _, node := range e.Cluster.shardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetValue(colID, value)
if err != nil {
return false, err
} else if val {
ret = true
}
default:
return ErrInvalidBSIGroupValueType
continue
}
field.Stats.Count("SetValue", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
return nil
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
continue
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
// Return first error.
for range nodes {
if err := <-resp; err != nil {
return err
// Forward call to remote node otherwise.
if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
}
}
return nil
return ret, nil
}
// executeSetRowAttrs executes a SetRowAttrs() call.
@ -1392,7 +1374,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
// exec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
pbreq := &QueryRequest{
Query: q.String(),
Shards: shards,
Remote: true,
@ -1403,40 +1385,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *
return nil, err
}
// Return an error, if specified on response.
if err := decodeError(pb.Err); err != nil {
return nil, err
}
// Return appropriate data for the query.
results = make([]interface{}, len(q.Calls))
for i, call := range q.Calls {
var v interface{}
var err error
switch call.Name {
case "Average", "Sum":
v, err = decodeValCount(pb.Results[i].GetValCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
v, err = pb.Results[i].N, nil
case "Set":
v, err = pb.Results[i].Changed, nil
case "Clear":
v, err = pb.Results[i].Changed, nil
case "SetRowAttrs":
case "SetColumnAttrs":
default:
v, err = DecodeRow(pb.Results[i].GetRow()), nil
}
if err != nil {
return nil, err
}
results[i] = v
}
return results, nil
return pb.Results, pb.Err
}
// shardsByNode returns a mapping of nodes to shards.
@ -1490,7 +1439,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
return nil, errors.Wrap(ctx.Err(), "context done")
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
@ -1500,10 +1449,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable {
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, err
return nil, errors.Wrap(err, "calling mapper")
}
continue
}
@ -1524,7 +1473,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return err
return errors.Wrap(err, "shards by node")
}
// Execute each node in a separate goroutine.
@ -1771,20 +1720,6 @@ func (vc *ValCount) Add(other ValCount) ValCount {
}
}
func EncodeValCount(vc ValCount) *internal.ValCount {
return &internal.ValCount{
Val: vc.Val,
Count: vc.Count,
}
}
func decodeValCount(pb *internal.ValCount) ValCount {
return ValCount{
Val: pb.Val,
Count: pb.Count,
}
}
// Smaller returns the smaller of the two ValCounts.
func (vc *ValCount) Smaller(other ValCount) ValCount {
if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) {

162
executor_test.go
View file

@ -39,7 +39,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateField("f", pilosa.FieldOptions{})
f, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
@ -112,7 +112,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
@ -354,7 +354,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
}
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
@ -365,7 +365,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Run("ErrInvalidRowValueType", func(t *testing.T) {
index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `row value must be a string when field 'keys' option enabled` {
@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) {
hldr.SetBit("i", "f", 1, 0)
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" {
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err))
t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err)
}
}
@ -398,20 +398,16 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
// Create felds.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 50,
}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
// Set bsiGroup values.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil {
t.Fatal(err)
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil {
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil {
t.Fatal(err)
}
@ -439,28 +435,24 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100,
}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil {
t.Fatal(err)
}
t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` {
t.Fatalf("unexpected error: %s", err)
}
})
@ -475,9 +467,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
@ -514,9 +506,9 @@ func TestExecutor_Execute_TopN(t *testing.T) {
// Set columns for rows 0, 10, & 20 across two shards.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
} else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set(0, f=0)
@ -555,9 +547,9 @@ func TestExecutor_Execute_TopN(t *testing.T) {
// Set columns for rows 0, 10, & 20 across two shards.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil {
} else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set("a", f="foo")
@ -741,15 +733,11 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -10,
Max: 100,
}); err != nil {
if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil {
t.Fatal(err)
}
@ -760,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
Set(1, x=1)
Set(` + strconv.Itoa(ShardWidth+2) + `, x=2)
SetValue(col=0, f=20)
SetValue(col=1, f=-5)
SetValue(col=2, f=-5)
SetValue(col=3, f=10)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60)
Set(0, f=20)
Set(1, f=-5)
Set(2, f=-5)
Set(3, f=10)
Set(` + strconv.Itoa(ShardWidth) + `, f=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, f=40)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=60)
`}); err != nil {
t.Fatal(err)
}
@ -836,31 +824,19 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("foo", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 100,
}); err != nil {
if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("bar", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100000,
}); err != nil {
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("other", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}); err != nil {
if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
@ -868,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) {
Set(0, x=0)
Set(` + strconv.Itoa(ShardWidth+1) + `, x=0)
SetValue(col=0, foo=20)
SetValue(col=0, bar=2000)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
SetValue(col=0, other=1000)
Set(0, foo=20)
Set(0, bar=2000)
Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50)
Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
Set(0, other=1000)
`}); err != nil {
t.Fatal(err)
}
@ -906,10 +882,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil {
t.Fatal(err)
}
@ -962,39 +935,23 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("foo", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 100,
}); err != nil {
if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("bar", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100000,
}); err != nil {
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("other", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}); err != nil {
if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("edge", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
}); err != nil {
if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil {
t.Fatal(err)
}
@ -1002,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
Set(0, f=0)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=0)
SetValue(col=50, foo=20)
SetValue(col=50, bar=2000)
SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30)
SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10)
SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60)
SetValue(col=0, other=1000)
SetValue(col=0, edge=100)
SetValue(col=1, edge=-100)
Set(50, foo=20)
Set(50, bar=2000)
Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
Set(0, other=1000)
Set(0, edge=100)
Set(1, edge=-100)
`}); err != nil {
t.Fatal(err)
}
@ -1268,7 +1225,7 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) {
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := index.CreateField("f", pilosa.FieldOptions{})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -1352,10 +1309,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) {
indexName := strings.ToLower(string(tt.quantum))
index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{})
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: tt.quantum,
}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil {
t.Fatal(err)
}
// Populate

85
field.go
View file

@ -75,9 +75,28 @@ type Field struct {
Logger Logger
}
// FieldOption is a functional option type for pilosa.FieldOptions.
// FieldOption is a functional option type for pilosa.fieldOptions.
type FieldOption func(fo *FieldOptions) error
func OptFieldKeys() FieldOption {
return func(fo *FieldOptions) error {
fo.Keys = true
return nil
}
}
func OptFieldTypeDefault() FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
fo.Type = FieldTypeSet
fo.CacheType = DefaultCacheType
fo.CacheSize = DefaultCacheSize
return nil
}
}
func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
@ -120,12 +139,19 @@ func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption {
}
// NewField returns a new instance of field.
func NewField(path, index, name string, options FieldOptions) (*Field, error) {
func NewField(path, index, name string, opts FieldOption) (*Field, error) {
err := validateName(name)
if err != nil {
return nil, err
}
// Apply functional option.
fo := FieldOptions{}
err = opts(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
f := &Field{
path: path,
index: index,
@ -138,7 +164,7 @@ func NewField(path, index, name string, options FieldOptions) (*Field, error) {
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
options: applyDefaultOptions(options),
options: applyDefaultOptions(fo),
Logger: NopLogger,
}
@ -178,13 +204,6 @@ func (f *Field) Type() string {
return f.options.Type
}
// CacheType returns the caching mode for the field.
func (f *Field) CacheType() string {
f.mu.RLock()
defer f.mu.RUnlock()
return f.options.CacheType
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Field) SetCacheSize(v uint32) error {
@ -605,7 +624,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) {
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&internal.CreateViewMessage{
&CreateViewMessage{
Index: f.index,
Field: f.name,
View: name,
@ -1073,25 +1092,6 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
return nil
}
// encodeFields converts a into its internal representation.
func encodeFields(a []*Field) []*internal.Field {
other := make([]*internal.Field, len(a))
for i := range a {
other[i] = encodeField(a[i])
}
return other
}
// encodeField converts f into its internal representation.
func encodeField(f *Field) *internal.Field {
fo := f.options
return &internal.Field{
Name: f.name,
Meta: fo.Encode(),
Views: f.viewNames(),
}
}
type fieldSlice []*Field
func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1102,7 +1102,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
type FieldInfo struct {
Name string `json:"name"`
Options FieldOptions `json:"options"`
Views []*viewInfo `json:"views,omitempty"`
Views []*ViewInfo `json:"views,omitempty"`
}
type fieldInfoSlice []*FieldInfo
@ -1119,7 +1119,7 @@ type FieldOptions struct {
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Keys bool `json:"keys,omitempty"`
Keys bool `json:"keys"`
}
// applyDefaultOptions returns a new FieldOptions object
@ -1155,21 +1155,6 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
}
}
func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions {
if options == nil {
return nil
}
return &FieldOptions{
Type: options.Type,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
Min: options.Min,
Max: options.Max,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Keys: options.Keys,
}
}
func (o *FieldOptions) MarshalJSON() ([]byte, error) {
switch o.Type {
case FieldTypeSet:
@ -1177,28 +1162,34 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
Type string `json:"type"`
CacheType string `json:"cacheType"`
CacheSize uint32 `json:"cacheSize"`
Keys bool `json:"keys"`
}{
o.Type,
o.CacheType,
o.CacheSize,
o.Keys,
})
case FieldTypeInt:
return json.Marshal(struct {
Type string `json:"type"`
Min int64 `json:"min"`
Max int64 `json:"max"`
Keys bool `json:"keys"`
}{
o.Type,
o.Min,
o.Max,
o.Keys,
})
case FieldTypeTime:
return json.Marshal(struct {
Type string `json:"type"`
TimeQuantum TimeQuantum `json:"timeQuantum"`
Keys bool `json:"keys"`
}{
o.Type,
o.TimeQuantum,
o.Keys,
})
}
return nil, errors.New("invalid field type")

18
field_internal_test.go
View file

@ -153,7 +153,7 @@ func TestBSIGroup_BaseValue(t *testing.T) {
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := MustOpenField(FieldOptions{})
f := MustOpenField(OptFieldTypeDefault())
defer f.Close()
viewName := viewStandard + "_v"
@ -190,12 +190,12 @@ type TestField struct {
}
// NewTestField returns a new instance of TestField d/0.
func NewTestField(options FieldOptions) *TestField {
func NewTestField(opts FieldOption) *TestField {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
field, err := NewField(path, "i", "f", options)
field, err := NewField(path, "i", "f", opts)
if err != nil {
panic(err)
}
@ -203,8 +203,8 @@ func NewTestField(options FieldOptions) *TestField {
}
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(options FieldOptions) *TestField {
f := NewTestField(options)
func MustOpenField(opts FieldOption) *TestField {
f := NewTestField(opts)
if err := f.Open(); err != nil {
panic(err)
}
@ -225,7 +225,7 @@ func (f *TestField) Reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = NewField(path, index, name, FieldOptions{})
f.Field, err = NewField(path, index, name, OptFieldTypeDefault())
if err != nil {
return err
}
@ -253,7 +253,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) {
// Ensure field can open and retrieve a view.
func TestField_CreateViewIfNotExists(t *testing.T) {
f := MustOpenField(FieldOptions{})
f := MustOpenField(OptFieldTypeDefault())
defer f.Close()
// Create view.
@ -278,7 +278,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
}
func TestField_SetTimeQuantum(t *testing.T) {
f := MustOpenField(FieldOptions{Type: FieldTypeTime})
f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
defer f.Close()
// Set & retrieve time quantum.
@ -297,7 +297,7 @@ func TestField_SetTimeQuantum(t *testing.T) {
}
func TestField_RowTime(t *testing.T) {
f := MustOpenField(FieldOptions{Type: FieldTypeTime})
f := MustOpenField(OptFieldTypeTime(TimeQuantum("")))
defer f.Close()
if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil {

34
field_test.go
View file

@ -28,11 +28,7 @@ func TestField_SetValue(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
if err != nil {
t.Fatal(err)
}
@ -65,11 +61,7 @@ func TestField_SetValue(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30))
if err != nil {
t.Fatal(err)
}
@ -102,9 +94,7 @@ func TestField_SetValue(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeSet,
})
f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
@ -119,11 +109,7 @@ func TestField_SetValue(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
if err != nil {
t.Fatal(err)
}
@ -138,11 +124,7 @@ func TestField_SetValue(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
if err != nil {
t.Fatal(err)
}
@ -159,7 +141,7 @@ func TestField_NameRestriction(t *testing.T) {
if err != nil {
panic(err)
}
field, err := pilosa.NewField(path, "i", ".meta", pilosa.FieldOptions{})
field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault())
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
@ -191,13 +173,13 @@ func TestField_NameValidation(t *testing.T) {
panic(err)
}
for _, name := range validFieldNames {
_, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{})
_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
_, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{})
_, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault())
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}

2
fragment.go
View file

@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
}
// Execute query.
queryRequest := &internal.QueryRequest{
queryRequest := &QueryRequest{
Query: buffers[k].String(),
Remote: true,
}

4
fragment_internal_test.go
View file

@ -750,7 +750,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
defer index.Close()
// Create field.
field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked, CacheSize: cacheSize})
field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, cacheSize))
if err != nil {
t.Fatal(err)
}
@ -916,7 +916,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
defer index.Close()
// Create field.
field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked})
field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize))
if err != nil {
t.Fatal(err)
}

80
gossip/gossip.go
View file

@ -15,6 +15,8 @@
package gossip
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"log"
@ -24,10 +26,8 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
@ -42,8 +42,8 @@ type GossipMemberSet struct {
broadcasts *memberlist.TransmitLimitedQueue
pserver pilosa.MemberServer
config *gossipConfig
papi *pilosa.API
config *gossipConfig
Logger pilosa.Logger
@ -145,9 +145,10 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption {
}
// NewGossipMemberSet returns a new instance of GossipMemberSet based on options.
func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
host := s.Node().URI.Host()
func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) {
host := api.Node().URI.Host
g := &GossipMemberSet{
papi: api,
Logger: pilosa.NopLogger,
}
@ -157,7 +158,7 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
return nil, errors.Wrap(err, "executing option")
}
}
ger := newGossipEventReceiver(g.logger, s)
ger := newGossipEventReceiver(g.logger, api)
g.gossipEventReceiver = ger
if g.transport == nil {
@ -189,11 +190,11 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = g.transport.Net
conf.Name = s.Node().ID
conf.BindAddr = s.Node().URI.Host()
conf.Name = api.Node().ID
conf.BindAddr = api.Node().URI.Host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = hostToIP(s.Node().URI.Host())
conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
@ -214,14 +215,12 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet
gossipSeeds: cfg.Seeds,
}
g.pserver = s
return g, nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.pserver.Node()))
buf, err := g.papi.Serializer.Marshal(g.papi.Node())
if err != nil {
g.Logger.Printf("marshal message error: %s", err)
return []byte{}
@ -232,14 +231,9 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte {
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *GossipMemberSet) NotifyMsg(b []byte) {
m, err := pilosa.UnmarshalMessage(b)
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b))
if err != nil {
g.Logger.Printf("unmarshal message error: %s", err)
return
}
if err := g.pserver.ReceiveMessage(m); err != nil {
g.Logger.Printf("receive message error: %s", err)
return
g.Logger.Printf("cluster message error: %s", err)
}
}
@ -252,14 +246,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte {
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *GossipMemberSet) LocalState(join bool) []byte {
pb, err := g.pserver.LocalStatus()
if err != nil {
g.Logger.Printf("error getting local state, err=%s", err)
return []byte{}
m := &pilosa.NodeStatus{
Node: g.papi.Node(),
MaxShards: g.papi.MaxShards(context.Background()),
Schema: &pilosa.Schema{Indexes: g.papi.Holder().Schema()},
}
// Marshal nodestate data to bytes.
buf, err := proto.Marshal(pb)
buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer)
if err != nil {
g.Logger.Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
@ -270,13 +264,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte {
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side's LocalState.
func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Unmarshal nodestate data.
var pb internal.NodeStatus
if err := proto.Unmarshal(buf, &pb); err != nil {
g.Logger.Printf("error unmarshalling nodestate data, err=%s", err)
return
}
err := g.pserver.HandleRemoteStatus(&pb)
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf))
if err != nil {
g.Logger.Printf("merge state error: %s", err)
}
@ -288,18 +276,18 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) {
// Care must be taken that events are processed in a timely manner from
// the channel, since this delegate will block until an event can be sent.
type gossipEventReceiver struct {
ch chan memberlist.NodeEvent
eventHandler *pilosa.Server
ch chan memberlist.NodeEvent
papi *pilosa.API
logger *log.Logger
}
// newGossipEventReceiver returns a new instance of GossipEventReceiver.
func newGossipEventReceiver(logger *log.Logger, pserver *pilosa.Server) *gossipEventReceiver {
func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver {
ger := &gossipEventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
logger: logger,
eventHandler: pserver,
ch: make(chan memberlist.NodeEvent, 1),
logger: logger,
papi: papi,
}
go ger.listen()
return ger
@ -333,16 +321,20 @@ func (g *gossipEventReceiver) listen() {
}
// Get the node from the event.Node meta data.
var n internal.Node
if err := proto.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta data")
var n pilosa.Node
if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta into node")
}
ne := &internal.NodeEventMessage{
Event: uint32(nodeEventType),
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: &n,
}
if err := g.eventHandler.ReceiveMessage(ne); err != nil {
buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer)
if err != nil {
panic(err)
}
if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil {
g.logger.Printf("receive event error: %s", err)
}
}

37
handler.go
View file

@ -75,3 +75,40 @@ func (n nopHandler) Close() error {
}
var NopHandler Handler = nopHandler{}
type ImportValueRequest struct {
Index string
Field string
Shard uint64
ColumnIDs []uint64
ColumnKeys []string
Values []int64
}
type ImportRequest struct {
Index string
Field string
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
}
type ImportResponse struct {
Err string
}
type BlockDataRequest struct {
Index string
Field string
View string
Shard uint64
Block uint64
}
type BlockDataResponse struct {
RowIDs []uint64
ColumnIDs []uint64
}

40
holder.go
View file

@ -27,7 +27,6 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
)
@ -217,7 +216,7 @@ func (h *Holder) Schema() []*IndexInfo {
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.views() {
fi.Views = append(fi.Views, &viewInfo{Name: view.name})
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Fields = append(di.Fields, fi)
@ -229,8 +228,24 @@ func (h *Holder) Schema() []*IndexInfo {
return a
}
// limitedSchema returns schema information for all indexes and fields.
func (h *Holder) limitedSchema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
di.Fields = append(di.Fields, fi)
}
sort.Sort(fieldInfoSlice(di.Fields))
a = append(a, di)
}
sort.Sort(indexInfoSlice(a))
return a
}
// applySchema applies an internal Schema to Holder.
func (h *Holder) applySchema(schema *internal.Schema) error {
func (h *Holder) applySchema(schema *Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
opt := IndexOptions{}
@ -240,14 +255,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
}
// Create fields that don't exist.
for _, f := range index.Fields {
opt := decodeFieldOptions(f.Meta)
field, err := idx.CreateFieldIfNotExists(f.Name, *opt)
field, err := idx.createFieldIfNotExists(f.Name, f.Options)
if err != nil {
return errors.Wrap(err, "creating field")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := field.createViewIfNotExists(v)
_, err := field.createViewIfNotExists(v.Name)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -257,20 +271,6 @@ func (h *Holder) applySchema(schema *internal.Schema) error {
return nil
}
// encodeMaxShards creates and internal representation of max shards.
func (h *Holder) encodeMaxShards() *internal.MaxShards {
return &internal.MaxShards{
Standard: h.maxShards(),
}
}
// encodeSchema creates an internal representation of schema.
func (h *Holder) encodeSchema() *internal.Schema {
return &internal.Schema{
Indexes: encodeIndexes(h.Indexes()),
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }

8
holder_internal_test.go
View file

@ -60,7 +60,7 @@ func newHolder() *tHolder {
// MustCreateFieldIfNotExists returns a given field. Panic on error.
func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field {
f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, FieldOptions{})
f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault())
if err != nil {
panic(err)
}
@ -105,7 +105,7 @@ func TestHolder_Optn(t *testing.T) {
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
t.Fatal(err)
@ -129,7 +129,7 @@ func TestHolder_Optn(t *testing.T) {
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.createViewIfNotExists(viewStandard); err != nil {
t.Fatal(err)
@ -154,7 +154,7 @@ func TestHolder_Optn(t *testing.T) {
if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if view, err := field.createViewIfNotExists(viewStandard); err != nil {
t.Fatal(err)

10
holder_test.go
View file

@ -98,7 +98,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -117,7 +117,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -135,7 +135,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -157,7 +157,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
@ -178,7 +178,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(0, 0, nil); err != nil {
t.Fatal(err)

71
http/client.go
View file

@ -27,22 +27,15 @@ import (
"sort"
"strconv"
"crypto/tls"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pkg/errors"
)
// ClientOptions represents the configuration for a InternalHTTPClient
type ClientOptions struct {
TLS *tls.Config
}
// InternalClient represents a client to the Pilosa cluster.
type InternalClient struct {
defaultURI *pilosa.URI
serializer pilosa.Serializer
// The client to use for HTTP communication.
HTTPClient *http.Client
@ -66,13 +59,11 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
return &InternalClient{
defaultURI: defaultURI,
serializer: proto.Serializer{},
HTTPClient: remoteClient,
}
}
// Host returns the host the client was initialized with.
func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI }
// MaxShardByIndex returns the number of shards on a server by index.
func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxShardByIndex(ctx)
@ -217,22 +208,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
}
// Query executes query against the index.
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
return c.QueryNode(ctx, c.defaultURI, index, queryRequest)
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
if index == "" {
return nil, pilosa.ErrIndexRequired
} else if queryRequest.Query == "" {
return nil, pilosa.ErrQueryRequired
}
// Encode request object.
buf, err := proto.Marshal(queryRequest)
buf, err := c.serializer.Marshal(queryRequest)
if err != nil {
return nil, errors.Wrap(err, "marshaling")
return nil, errors.Wrap(err, "marshaling queryRequest")
}
// Create HTTP request.
@ -262,11 +252,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
return nil, errors.New(string(body))
}
qresp := &internal.QueryResponse{}
if err := proto.Unmarshal(body, qresp); err != nil {
qresp := &pilosa.QueryResponse{}
if err := c.serializer.Unmarshal(body, qresp); err != nil {
return nil, fmt.Errorf("unmarshal response: %s", err)
} else if s := qresp.Err; s != "" {
return nil, errors.New(s)
} else if qresp.Err != nil {
return nil, qresp.Err
}
return qresp, nil
@ -280,7 +270,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard
return pilosa.ErrFieldRequired
}
buf, err := marshalImportPayload(index, field, shard, bits)
buf, err := c.marshalImportPayload(index, field, shard, bits)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -309,7 +299,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum
return pilosa.ErrFieldRequired
}
buf, err := marshalImportPayloadK(index, field, columns)
buf, err := c.marshalImportPayloadK(index, field, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -343,14 +333,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel
}
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
Index: index,
Field: field,
Shard: shard,
@ -365,14 +355,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit)
}
// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowKeys := Bits(bits).RowKeys()
columnKeys := Bits(bits).ColumnKeys()
timestamps := Bits(bits).Timestamps()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportRequest{
Index: index,
Field: field,
RowKeys: rowKeys,
@ -414,8 +404,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde
return errors.New(string(body))
}
var isresp internal.ImportResponse
if err := proto.Unmarshal(body, &isresp); err != nil {
var isresp pilosa.ImportResponse
if err := c.serializer.Unmarshal(body, &isresp); err != nil {
return fmt.Errorf("unmarshal import response: %s", err)
} else if s := isresp.Err; s != "" {
return errors.New(s)
@ -432,7 +422,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
return pilosa.ErrFieldRequired
}
buf, err := marshalImportValuePayload(index, field, shard, vals)
buf, err := c.marshalImportValuePayload(index, field, shard, vals)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -454,13 +444,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
}
// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportValueRequest{
buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{
Index: index,
Field: field,
Shard: shard,
@ -683,7 +673,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
if uri == nil {
panic("need to pass a URI to BlockData")
}
buf, err := proto.Marshal(&internal.BlockDataRequest{
buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{
Index: index,
Field: field,
Shard: shard,
@ -719,10 +709,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
}
// Decode response object.
var rsp internal.BlockDataResponse
var rsp pilosa.BlockDataResponse
if body, err := ioutil.ReadAll(resp.Body); err != nil {
return nil, nil, errors.Wrap(err, "reading")
} else if err := proto.Unmarshal(body, &rsp); err != nil {
} else if err := c.serializer.Unmarshal(body, &rsp); err != nil {
return nil, nil, errors.Wrap(err, "unmarshalling")
}
return rsp.RowIDs, rsp.ColumnIDs, nil
@ -819,12 +809,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
}
// SendMessage posts a message synchronously.
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
u := uriPathToURL(uri, "/internal/cluster/message")
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
if err != nil {
@ -998,7 +983,7 @@ func pos(rowID, columnID uint64) uint64 {
func uriPathToURL(uri *pilosa.URI, path string) url.URL {
return url.URL{
Scheme: uri.Scheme(),
Scheme: uri.Scheme,
Host: uri.HostPort(),
Path: path,
}
@ -1006,7 +991,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL {
func nodePathToURL(node *pilosa.Node, path string) url.URL {
return url.URL{
Scheme: node.URI.Scheme(),
Scheme: node.URI.Scheme,
Host: node.URI.HostPort(),
Path: path,
}

16
http/client_test.go
View file

@ -24,7 +24,6 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) {
client[2] = MustNewClient(c[2].URL(), defaultClient)
topN := 4
queryRequest := &internal.QueryRequest{
queryRequest := &pilosa.QueryRequest{
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) {
}
// Test must return exactly N results.
if len(result.Results[0].Pairs) != topN {
if len(result.Results[0].([]pilosa.Pair)) != topN {
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
}
p := []*internal.Pair{
p := []pilosa.Pair{
{ID: 100, Count: 12},
{ID: 22, Count: 10},
{ID: 98, Count: 8},
{ID: 99, Count: 7}}
// Valdidate the Top 4 result counts.
if !reflect.DeepEqual(result.Results[0].Pairs, p) {
if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}
@ -218,15 +217,10 @@ func TestClient_ImportValue(t *testing.T) {
hldr := test.Holder{Holder: holder}
fldName := "f"
fo := pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
}
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
field, err := index.CreateFieldIfNotExists(fldName, fo)
field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
if err != nil {
t.Fatal(err)
}

115
http/handler.go
View file

@ -33,11 +33,9 @@ import (
"strings"
"time"
"github.com/gogo/protobuf/proto"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -358,9 +356,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
}
schema := h.API.Schema(r.Context())
if err := json.NewEncoder(w).Encode(getSchemaResponse{
Indexes: schema,
}); err != nil {
if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil {
h.Logger.Printf("write schema response error: %s", err)
}
}
@ -374,7 +370,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
status := getStatusResponse{
State: h.API.State(),
Nodes: h.API.Hosts(r.Context()),
LocalID: h.API.LocalID(),
LocalID: h.API.Node().ID,
}
if err := json.NewEncoder(w).Encode(status); err != nil {
h.Logger.Printf("write status response error: %s", err)
@ -416,18 +412,25 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
resp, err := h.API.Query(r.Context(), req)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
switch errors.Cause(resp.Err) {
case pilosa.ErrTooManyWrites:
w.WriteHeader(http.StatusRequestEntityTooLarge)
default:
w.WriteHeader(http.StatusBadRequest)
}
h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err})
return
}
// Set appropriate status code, if there is an error.
// Set appropriate status code, if there is an error. It doesn't appear that
// resp.Err could ever be set in API.Query, so this code block is probably
// doing nothing right now.
if resp.Err != nil {
switch resp.Err {
switch errors.Cause(resp.Err) {
case pilosa.ErrTooManyWrites:
w.WriteHeader(http.StatusRequestEntityTooLarge)
default:
w.WriteHeader(http.StatusInternalServerError)
w.WriteHeader(http.StatusBadRequest)
}
}
@ -655,17 +658,22 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
}
// Convert json options into functional options.
var fos pilosa.FieldOption
var fos []pilosa.FieldOption
switch req.Options.Type {
case pilosa.FieldTypeSet:
fos = pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)
fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize))
case pilosa.FieldTypeInt:
fos = pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)
fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max))
case pilosa.FieldTypeTime:
fos = pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
}
if req.Options.Keys != nil {
if *req.Options.Keys {
fos = append(fos, pilosa.OptFieldKeys())
}
}
_, err = h.API.CreateField(r.Context(), indexName, fieldName, fos)
_, err = h.API.CreateField(r.Context(), indexName, fieldName, fos...)
resp.write(w, err)
}
@ -817,13 +825,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques
return nil, errors.Wrap(err, "reading")
}
// Unmarshal into object.
var req internal.QueryRequest
if err := proto.Unmarshal(body, &req); err != nil {
return nil, errors.Wrap(err, "unmarshalling")
qreq := &pilosa.QueryRequest{}
err = h.API.Serializer.Unmarshal(body, qreq)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling query request")
}
return decodeQueryRequest(&req), nil
return qreq, nil
}
// readURLQueryRequest parses query parameters from URL parameters from r.
@ -862,7 +869,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res
// writeProtobufQueryResponse writes the response from the executor to w as protobuf.
func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error {
if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil {
if buf, err := h.API.Serializer.Marshal(resp); err != nil {
return errors.Wrap(err, "marshalling")
} else if _, err := w.Write(buf); err != nil {
return errors.Wrap(err, "writing")
@ -914,8 +921,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
if field.Type() == pilosa.FieldTypeInt {
// Field type: Int
// Marshal into request object.
var req internal.ImportValueRequest
if err := proto.Unmarshal(body, &req); err != nil {
req := &pilosa.ImportValueRequest{}
if err := h.API.Serializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
@ -932,8 +939,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
} else {
// Field type: Set, Time
// Marshal into request object.
var req internal.ImportRequest
if err := proto.Unmarshal(body, &req); err != nil {
req := &pilosa.ImportRequest{}
if err := h.API.Serializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
@ -950,7 +957,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Marshal response object.
buf, e := proto.Marshal(&internal.ImportResponse{Err: ""})
buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""})
if e != nil {
http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError)
return
@ -1105,56 +1112,6 @@ const (
QueryResultTypeBool
)
func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest {
req := &pilosa.QueryRequest{
Query: pb.Query,
Shards: pb.Shards,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,
ExcludeColumns: pb.ExcludeColumns,
}
return req
}
func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets),
}
for i := range resp.Results {
pb.Results[i] = &internal.QueryResult{}
switch result := resp.Results[i].(type) {
case *pilosa.Row:
pb.Results[i].Type = QueryResultTypeRow
pb.Results[i].Row = pilosa.EncodeRow(result)
case []pilosa.Pair:
pb.Results[i].Type = QueryResultTypePairs
pb.Results[i].Pairs = pilosa.EncodePairs(result)
case pilosa.ValCount:
pb.Results[i].Type = QueryResultTypeValCount
pb.Results[i].ValCount = pilosa.EncodeValCount(result)
case uint64:
pb.Results[i].Type = QueryResultTypeUint64
pb.Results[i].N = result
case bool:
pb.Results[i].Type = QueryResultTypeBool
pb.Results[i].Changed = result
case nil:
pb.Results[i].Type = QueryResultTypeNil
}
}
if resp.Err != nil {
pb.Err = resp.Err.Error()
}
return pb
}
// parseUint64Slice returns a slice of uint64s from a comma-delimited string.
func parseUint64Slice(s string) ([]uint64, error) {
var a []uint64
@ -1326,10 +1283,6 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
}
}
func (h *Handler) GetAPI() *pilosa.API {
return h.API
}
type defaultClusterMessageResponse struct{}
func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {

6
http/translator_test.go
View file

@ -75,7 +75,7 @@ func TestTranslateStore_Reader(t *testing.T) {
defer main.Close()
// Connect to server and stream all available data.
store := http.NewTranslateStore(main.Server.URI.String())
store := http.NewTranslateStore(main.URL())
rc, err := store.Reader(context.Background(), 100)
if err != nil {
@ -128,7 +128,7 @@ func TestTranslateStore_Reader(t *testing.T) {
// Connect to server and begin streaming.
ctx, cancel := context.WithCancel(context.Background())
store := http.NewTranslateStore(main.Server.URI.String())
store := http.NewTranslateStore(main.URL())
if _, err := store.Reader(ctx, 0); err != nil {
t.Fatal(err)
}
@ -155,7 +155,7 @@ func TestTranslateStore_Reader(t *testing.T) {
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
_, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0)
_, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0)
if err != pilosa.ErrNotImplemented {
t.Fatalf("unexpected error: %s", err)
}

66
index.go
View file

@ -270,7 +270,7 @@ func (i *Index) RecalculateCaches() {
}
// CreateField creates a field.
func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
@ -278,11 +278,40 @@ func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
if i.fields[name] != nil {
return nil, NewConflictError(ErrFieldExists)
}
return i.createField(name, opt)
// Apply functional options.
fo := FieldOptions{}
for _, opt := range opts {
err := opt(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
}
return i.createField(name, fo)
}
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find field in cache first.
if f := i.fields[name]; f != nil {
return f, nil
}
// Apply functional option.
fo := FieldOptions{}
err := opts(&fo)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
return i.createField(name, fo)
}
func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
@ -330,7 +359,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
}
func (i *Index) newField(path, name string) (*Field, error) {
f, err := NewField(path, i.name, name, FieldOptions{}) // TODO: NewField should be un-exported along with FieldOptions
f, err := NewField(path, i.name, name, OptFieldTypeDefault())
if err != nil {
return nil, err
}
@ -376,8 +405,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
Fields []*FieldInfo `json:"fields"`
Name string `json:"name"`
Options IndexOptions `json:"options"`
Fields []*FieldInfo `json:"fields"`
}
type indexInfoSlice []*IndexInfo
@ -386,35 +416,11 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexInfoSlice) Len() int { return len(p) }
func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// encodeIndexes converts a into its internal representation.
func encodeIndexes(a []*Index) []*internal.Index {
other := make([]*internal.Index, len(a))
for i := range a {
other[i] = encodeIndex(a[i])
}
return other
}
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Fields: encodeFields(d.Fields()),
}
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
Keys bool `json:"keys"`
}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
Keys: i.Keys,
}
}
// hasTime returns true if a contains a non-nil time.
func hasTime(a []*time.Time) bool {
for _, t := range a {

17
index_test.go
View file

@ -33,7 +33,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) {
defer index.Close()
// Create field.
f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
f, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
} else if f == nil {
@ -41,7 +41,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) {
}
// Retrieve existing field.
other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
other, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
} else if f.Field != other.Field {
@ -61,10 +61,7 @@ func TestIndex_CreateField(t *testing.T) {
defer index.Close()
// Create field with explicit quantum.
f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
})
f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
@ -80,11 +77,7 @@ func TestIndex_CreateField(t *testing.T) {
defer index.Close()
// Create field with schema and verify it exists.
if f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 20,
}); err != nil {
if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
t.Fatalf("unexpected type: %#v", f.Type())
@ -184,7 +177,7 @@ func TestIndex_DeleteField(t *testing.T) {
defer index.Close()
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}

0
iterator_test.go → iterator_internal_test.go
View file

19
pilosa.go
View file

@ -17,8 +17,6 @@ package pilosa
import (
"errors"
"regexp"
"github.com/pilosa/pilosa/internal"
)
// System errors.
@ -122,23 +120,6 @@ type ColumnAttrSet struct {
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
// EncodeColumnAttrSets converts a into its internal representation.
func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
other := make([]*internal.ColumnAttrSet, len(a))
for i := range a {
other[i] = EncodeColumnAttrSet(a[i])
}
return other
}
// EncodeColumnAttrSet converts set into its internal representation.
func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
return &internal.ColumnAttrSet{
ID: set.ID,
Attrs: encodeAttrs(set.Attrs),
}
}
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"

53
pql/ast.go
View file

@ -220,23 +220,6 @@ func (q *Query) WriteCallN() int {
return n
}
// HasKeys returns true if any call in the query uses keys and requires translation to ids.
func (q *Query) HasKeys() bool {
for _, call := range q.Calls {
if call.Args["col"] != nil {
if _, ok := call.Args["col"].(string); ok {
return true
}
}
if call.Args["row"] != nil {
if _, ok := call.Args["row"].(string); ok {
return true
}
}
}
return false
}
// String returns a string representation of the query.
func (q *Query) String() string {
a := make([]string, len(q.Calls))
@ -285,6 +268,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
}
}
// IntArg is for reading the value at key from call.Args as an int64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a unt64 or an int64 and
// then cast to an int64. An error is returned if the value is not an int64 or
// uint64.
func (c *Call) IntArg(key string) (int64, bool, error) {
val, ok := c.Args[key]
if !ok {
return 0, false, nil
}
switch tval := val.(type) {
case int64:
return tval, true, nil
case uint64:
return int64(tval), true, nil
default:
return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval)
}
}
// UintSliceArg reads the value at key from call.Args as a slice of uint64. If
// the key is not in Call.Args, the value of the returned bool will be false,
// and the error will be nil. If the value is a slice of int64 it will convert
@ -309,22 +312,6 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
}
}
// StringArg is for reading the value at key from call.Args as a string. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. An error is returned if the value is not a string.
func (c *Call) StringArg(key string) (string, bool, error) {
val, ok := c.Args[key]
if !ok {
return "", false, nil
}
switch tval := val.(type) {
case string:
return tval, true, nil
default:
return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval)
}
}
// Keys returns a list of argument keys in sorted order.
func (c *Call) Keys() []string {
a := make([]string, 0, len(c.Args))

7
pql/pql.peg
View file

@ -5,7 +5,7 @@ type PQL Peg {
}
Calls <- whitesp (Call whitesp)* !.
Calls <- sp (Call sp)* !.
Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
/ 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()}
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
@ -62,11 +62,10 @@ col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
open <- '(' sp
close <- ')' sp
sp <- ( ' ' / '\t' )*
comma <- sp ',' whitesp
sp <- ( ' ' / '\t' / '\n' )*
comma <- sp ',' sp
lbrack <- '[' sp
rbrack <- sp ']' sp
whitesp <- ( ' ' / '\t' / '\n' )*
IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*

266
pql/pql.peg.go
View file

@ -45,7 +45,6 @@ const (
rulecomma
rulelbrack
rulerbrack
rulewhitesp
ruleIDENT
ruletimestampbasicfmt
ruletimestampfmt
@ -128,7 +127,6 @@ var rul3s = [...]string{
"comma",
"lbrack",
"rbrack",
"whitesp",
"IDENT",
"timestampbasicfmt",
"timestampfmt",
@ -294,7 +292,7 @@ type PQL struct {
Buffer string
buffer []rune
rules [80]func() bool
rules [79]func() bool
parse func(rule ...int) error
reset func()
Pretty bool
@ -548,12 +546,12 @@ func (p *PQL) Init() {
_rules = [...]func() bool{
nil,
/* 0 Calls <- <(whitesp (Call whitesp)* !.)> */
/* 0 Calls <- <(sp (Call sp)* !.)> */
func() bool {
position0, tokenIndex0 := position, tokenIndex
{
position1 := position
if !_rules[rulewhitesp]() {
if !_rules[rulesp]() {
goto l0
}
l2:
@ -562,7 +560,7 @@ func (p *PQL) Init() {
if !_rules[ruleCall]() {
goto l3
}
if !_rules[rulewhitesp]() {
if !_rules[rulesp]() {
goto l3
}
goto l2
@ -2525,7 +2523,7 @@ func (p *PQL) Init() {
position, tokenIndex = position257, tokenIndex257
return false
},
/* 25 sp <- <(' ' / '\t')*> */
/* 25 sp <- <(' ' / '\t' / '\n')*> */
func() bool {
{
position260 := position
@ -2542,6 +2540,13 @@ func (p *PQL) Init() {
l264:
position, tokenIndex = position263, tokenIndex263
if buffer[position] != rune('\t') {
goto l265
}
position++
goto l263
l265:
position, tokenIndex = position263, tokenIndex263
if buffer[position] != rune('\n') {
goto l262
}
position++
@ -2555,295 +2560,258 @@ func (p *PQL) Init() {
}
return true
},
/* 26 comma <- <(sp ',' whitesp)> */
/* 26 comma <- <(sp ',' sp)> */
func() bool {
position265, tokenIndex265 := position, tokenIndex
position266, tokenIndex266 := position, tokenIndex
{
position266 := position
position267 := position
if !_rules[rulesp]() {
goto l265
goto l266
}
if buffer[position] != rune(',') {
goto l265
goto l266
}
position++
if !_rules[rulewhitesp]() {
goto l265
if !_rules[rulesp]() {
goto l266
}
add(rulecomma, position266)
add(rulecomma, position267)
}
return true
l265:
position, tokenIndex = position265, tokenIndex265
l266:
position, tokenIndex = position266, tokenIndex266
return false
},
/* 27 lbrack <- <('[' sp)> */
nil,
/* 28 rbrack <- <(sp ']' sp)> */
nil,
/* 29 whitesp <- <(' ' / '\t' / '\n')*> */
func() bool {
{
position270 := position
l271:
{
position272, tokenIndex272 := position, tokenIndex
{
position273, tokenIndex273 := position, tokenIndex
if buffer[position] != rune(' ') {
goto l274
}
position++
goto l273
l274:
position, tokenIndex = position273, tokenIndex273
if buffer[position] != rune('\t') {
goto l275
}
position++
goto l273
l275:
position, tokenIndex = position273, tokenIndex273
if buffer[position] != rune('\n') {
goto l272
}
position++
}
l273:
goto l271
l272:
position, tokenIndex = position272, tokenIndex272
}
add(rulewhitesp, position270)
}
return true
},
/* 30 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */
/* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */
nil,
/* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */
/* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */
func() bool {
position277, tokenIndex277 := position, tokenIndex
position271, tokenIndex271 := position, tokenIndex
{
position278 := position
position272 := position
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if buffer[position] != rune('-') {
goto l277
goto l271
}
position++
{
position279, tokenIndex279 := position, tokenIndex
position273, tokenIndex273 := position, tokenIndex
if buffer[position] != rune('0') {
goto l280
goto l274
}
position++
goto l279
l280:
position, tokenIndex = position279, tokenIndex279
goto l273
l274:
position, tokenIndex = position273, tokenIndex273
if buffer[position] != rune('1') {
goto l277
goto l271
}
position++
}
l279:
l273:
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if buffer[position] != rune('-') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('3') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if buffer[position] != rune('T') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if buffer[position] != rune(':') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l277
goto l271
}
position++
add(ruletimestampbasicfmt, position278)
add(ruletimestampbasicfmt, position272)
}
return true
l277:
position, tokenIndex = position277, tokenIndex277
l271:
position, tokenIndex = position271, tokenIndex271
return false
},
/* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */
/* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */
func() bool {
position281, tokenIndex281 := position, tokenIndex
position275, tokenIndex275 := position, tokenIndex
{
position282 := position
position276 := position
{
position283, tokenIndex283 := position, tokenIndex
position277, tokenIndex277 := position, tokenIndex
if buffer[position] != rune('"') {
goto l284
goto l278
}
position++
if !_rules[ruletimestampbasicfmt]() {
goto l284
goto l278
}
if buffer[position] != rune('"') {
goto l284
goto l278
}
position++
goto l283
l284:
position, tokenIndex = position283, tokenIndex283
goto l277
l278:
position, tokenIndex = position277, tokenIndex277
if buffer[position] != rune('\'') {
goto l285
goto l279
}
position++
if !_rules[ruletimestampbasicfmt]() {
goto l285
goto l279
}
if buffer[position] != rune('\'') {
goto l285
goto l279
}
position++
goto l283
l285:
position, tokenIndex = position283, tokenIndex283
goto l277
l279:
position, tokenIndex = position277, tokenIndex277
if !_rules[ruletimestampbasicfmt]() {
goto l281
goto l275
}
}
l283:
add(ruletimestampfmt, position282)
l277:
add(ruletimestampfmt, position276)
}
return true
l281:
position, tokenIndex = position281, tokenIndex281
l275:
position, tokenIndex = position275, tokenIndex275
return false
},
/* 33 timestamp <- <(<timestampfmt> Action43)> */
/* 32 timestamp <- <(<timestampfmt> Action43)> */
nil,
/* 35 Action0 <- <{p.startCall("Set")}> */
/* 34 Action0 <- <{p.startCall("Set")}> */
nil,
/* 36 Action1 <- <{p.endCall()}> */
/* 35 Action1 <- <{p.endCall()}> */
nil,
/* 37 Action2 <- <{p.startCall("SetRowAttrs")}> */
/* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */
nil,
/* 38 Action3 <- <{p.endCall()}> */
/* 37 Action3 <- <{p.endCall()}> */
nil,
/* 39 Action4 <- <{p.startCall("SetColumnAttrs")}> */
/* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */
nil,
/* 40 Action5 <- <{p.endCall()}> */
/* 39 Action5 <- <{p.endCall()}> */
nil,
/* 41 Action6 <- <{p.startCall("Clear")}> */
/* 40 Action6 <- <{p.startCall("Clear")}> */
nil,
/* 42 Action7 <- <{p.endCall()}> */
/* 41 Action7 <- <{p.endCall()}> */
nil,
/* 43 Action8 <- <{p.startCall("TopN")}> */
/* 42 Action8 <- <{p.startCall("TopN")}> */
nil,
/* 44 Action9 <- <{p.endCall()}> */
/* 43 Action9 <- <{p.endCall()}> */
nil,
/* 45 Action10 <- <{p.startCall("Range")}> */
/* 44 Action10 <- <{p.startCall("Range")}> */
nil,
/* 46 Action11 <- <{p.endCall()}> */
/* 45 Action11 <- <{p.endCall()}> */
nil,
nil,
/* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */
/* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */
nil,
/* 49 Action13 <- <{ p.endCall() }> */
/* 48 Action13 <- <{ p.endCall() }> */
nil,
/* 50 Action14 <- <{ p.addBTWN() }> */
/* 49 Action14 <- <{ p.addBTWN() }> */
nil,
/* 51 Action15 <- <{ p.addLTE() }> */
/* 50 Action15 <- <{ p.addLTE() }> */
nil,
/* 52 Action16 <- <{ p.addGTE() }> */
/* 51 Action16 <- <{ p.addGTE() }> */
nil,
/* 53 Action17 <- <{ p.addEQ() }> */
/* 52 Action17 <- <{ p.addEQ() }> */
nil,
/* 54 Action18 <- <{ p.addNEQ() }> */
/* 53 Action18 <- <{ p.addNEQ() }> */
nil,
/* 55 Action19 <- <{ p.addLT() }> */
/* 54 Action19 <- <{ p.addLT() }> */
nil,
/* 56 Action20 <- <{ p.addGT() }> */
/* 55 Action20 <- <{ p.addGT() }> */
nil,
/* 57 Action21 <- <{p.startConditional()}> */
/* 56 Action21 <- <{p.startConditional()}> */
nil,
/* 58 Action22 <- <{p.endConditional()}> */
/* 57 Action22 <- <{p.endConditional()}> */
nil,
/* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */
/* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */
nil,
/* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */
/* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */
nil,
/* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */
/* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */
nil,
/* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */
/* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */
nil,
/* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */
/* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */
nil,
/* 64 Action28 <- <{ p.startList() }> */
/* 63 Action28 <- <{ p.startList() }> */
nil,
/* 65 Action29 <- <{ p.endList() }> */
/* 64 Action29 <- <{ p.endList() }> */
nil,
/* 66 Action30 <- <{ p.addVal(nil) }> */
/* 65 Action30 <- <{ p.addVal(nil) }> */
nil,
/* 67 Action31 <- <{ p.addVal(true) }> */
/* 66 Action31 <- <{ p.addVal(true) }> */
nil,
/* 68 Action32 <- <{ p.addVal(false) }> */
/* 67 Action32 <- <{ p.addVal(false) }> */
nil,
/* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */
/* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */
nil,
/* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */
/* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */
nil,
/* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */
/* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */
nil,
/* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */
/* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */
nil,
/* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */
/* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */
nil,
/* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */
/* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */
nil,
/* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */
/* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */
nil,
/* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */
/* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */
nil,
/* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */
/* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */
nil,
/* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */
/* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */
nil,
/* 79 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */
/* 78 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */
nil,
}
p.rules = _rules

7
pql/pqlpeg_test.go
View file

@ -228,6 +228,13 @@ func TestPEGWorking(t *testing.T) {
name: "Dashed Frame",
input: "Set(1, my-frame=9)",
ncalls: 1},
{
name: "newlines",
input: `Set(
1,
my-frame
=9)`,
ncalls: 1},
}
for i, test := range tests {

55
row.go
View file

@ -18,7 +18,6 @@ import (
"encoding/json"
"sort"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/roaring"
)
@ -216,25 +215,6 @@ func (r *Row) InvalidateCount() {
}
}
// IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased.
func (r *Row) IncrementCount(i uint64) {
seg := r.segment(i / ShardWidth)
if seg != nil {
seg.n++
}
}
// DecrementCount decrements the row cached counter.
func (r *Row) DecrementCount(i uint64) {
seg := r.segment(i / ShardWidth)
if seg != nil {
if seg.n > 0 {
seg.n--
}
}
}
// Count returns the number of columns in the row.
func (r *Row) Count() uint64 {
var n uint64
@ -271,41 +251,6 @@ func (r *Row) Columns() []uint64 {
return a
}
// EncodeRow converts r into its internal representation.
func EncodeRow(r *Row) *internal.Row {
if r == nil {
return nil
}
return &internal.Row{
Columns: r.Columns(),
Attrs: encodeAttrs(r.Attrs),
}
}
// DecodeRow converts r from its internal representation.
func DecodeRow(pr *internal.Row) *Row {
if pr == nil {
return nil
}
r := NewRow()
r.Attrs = decodeAttrs(pr.Attrs)
for _, v := range pr.Columns {
r.SetBit(v)
}
return r
}
// Union performs a union on a slice of rows.
func Union(rows []*Row) *Row {
other := rows[0]
for _, r := range rows[1:] {
other = other.Union(r)
}
return other
}
// RowSegment holds a subset of a row.
// This could point to a mmapped roaring bitmap or an in-memory bitmap. The
// width of the segment will always match the shard width.

149
server.go
View file

@ -27,8 +27,6 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -36,12 +34,11 @@ import (
// Default server settings.
const (
DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
defaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics"
)
// Ensure Server implements interfaces.
var _ broadcaster = &Server{}
var _ MemberServer = &Server{}
// Server represents a holder wrapped by a running HTTP server.
type Server struct {
@ -57,6 +54,7 @@ type Server struct {
executor *executor
hosts []string
clusterDisabled bool
serializer Serializer
// External
systemInfo SystemInfo
@ -64,7 +62,7 @@ type Server struct {
logger Logger
nodeID string
URI URI
uri URI
antiEntropyInterval time.Duration
metricInterval time.Duration
diagnosticInterval time.Duration
@ -188,7 +186,7 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
func OptServerURI(uri *URI) ServerOption {
return func(s *Server) error {
s.URI = *uri
s.uri = *uri
return nil
}
}
@ -203,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption {
}
}
func OptServerSerializer(ser Serializer) ServerOption {
return func(s *Server) error {
s.serializer = ser
return nil
}
}
func OptServerIsCoordinator(is bool) ServerOption {
return func(s *Server) error {
s.isCoordinator = is
@ -230,7 +235,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
closing: make(chan struct{}),
cluster: newCluster(),
holder: NewHolder(),
diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
diagnostics: NewDiagnosticsCollector(defaultDiagnosticServer),
systemInfo: NewNopSystemInfo(),
gcNotifier: NopGCNotifier,
@ -277,7 +282,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
// Set Cluster Node.
node := &Node{
ID: s.nodeID,
URI: s.URI,
URI: s.uri,
IsCoordinator: s.cluster.Coordinator == s.nodeID,
}
s.cluster.Node = node
@ -431,41 +436,41 @@ func (s *Server) monitorAntiEntropy() {
}
}
// ReceiveMessage represents an implementation of BroadcastHandler.
func (s *Server) ReceiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.CreateShardMessage:
// receiveMessage represents an implementation of BroadcastHandler.
func (s *Server) receiveMessage(m Message) error {
switch obj := m.(type) {
case *CreateShardMessage:
idx := s.holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
idx.setRemoteMaxShard(obj.Shard)
case *internal.CreateIndexMessage:
case *CreateIndexMessage:
opt := IndexOptions{}
_, err := s.holder.CreateIndex(obj.Index, opt)
if err != nil {
return err
}
case *internal.DeleteIndexMessage:
case *DeleteIndexMessage:
if err := s.holder.DeleteIndex(obj.Index); err != nil {
return err
}
case *internal.CreateFieldMessage:
case *CreateFieldMessage:
idx := s.holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := decodeFieldOptions(obj.Meta)
_, err := idx.CreateField(obj.Field, *opt)
opt := obj.Meta
_, err := idx.createField(obj.Field, *opt)
if err != nil {
return err
}
case *internal.DeleteFieldMessage:
case *DeleteFieldMessage:
idx := s.holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateViewMessage:
case *CreateViewMessage:
f := s.holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Field not found: %s", obj.Field)
@ -474,7 +479,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.DeleteViewMessage:
case *DeleteViewMessage:
f := s.holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Field not found: %s", obj.Field)
@ -483,52 +488,59 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.ClusterStatus:
case *ClusterStatus:
err := s.cluster.mergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
case *ResizeInstruction:
err := s.cluster.followResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
case *ResizeInstructionComplete:
err := s.cluster.markResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.cluster.setCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.cluster.updateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
case *SetCoordinatorMessage:
s.cluster.setCoordinator(obj.New)
case *UpdateCoordinatorMessage:
s.cluster.updateCoordinator(obj.New)
case *NodeStateMessage:
err := s.cluster.receiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
case *internal.RecalculateCaches:
case *RecalculateCaches:
s.holder.RecalculateCaches()
case *internal.NodeEventMessage:
s.cluster.ReceiveEvent(DecodeNodeEvent(obj))
case *NodeEvent:
s.cluster.ReceiveEvent(obj)
case *NodeStatus:
s.handleRemoteStatus(obj)
}
return nil
}
// SendSync represents an implementation of Broadcaster.
func (s *Server) SendSync(pb proto.Message) error {
func (s *Server) SendSync(m Message) error {
var eg errgroup.Group
msg, err := s.serializer.Marshal(m)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
msg = append([]byte{getMessageType(m)}, msg...)
for _, node := range s.cluster.Nodes {
node := node
s.logger.Printf("SendSync to: %s", node.URI)
// Don't forward the message to ourselves.
if s.URI == node.URI {
if s.uri == node.URI {
continue
}
eg.Go(func() error {
return s.defaultClient.SendMessage(context.Background(), &node.URI, pb)
return s.defaultClient.SendMessage(context.Background(), &node.URI, msg)
})
}
@ -536,72 +548,48 @@ func (s *Server) SendSync(pb proto.Message) error {
}
// SendAsync represents an implementation of Broadcaster.
func (s *Server) SendAsync(pb proto.Message) error {
func (s *Server) SendAsync(m Message) error {
return ErrNotImplemented
}
// SendTo represents an implementation of Broadcaster.
func (s *Server) SendTo(to *Node, pb proto.Message) error {
func (s *Server) SendTo(to *Node, m Message) error {
s.logger.Printf("SendTo: %s", to.URI)
return s.defaultClient.SendMessage(context.Background(), &to.URI, pb)
msg, err := s.serializer.Marshal(m)
if err != nil {
return fmt.Errorf("marshaling message: %v", err)
}
msg = append([]byte{getMessageType(m)}, msg...)
return s.defaultClient.SendMessage(context.Background(), &to.URI, msg)
}
// Node returns the pilosa.Node object. It is used by membership protocols to
// node returns the pilosa.node object. It is used by membership protocols to
// get this node's name(ID), location(URI), and coordinator status.
func (s *Server) Node() *Node {
return s.cluster.Node
func (s *Server) node() Node {
return *s.cluster.Node
}
// Server implements StatusHandler.
// LocalStatus is used to periodically sync information
// between nodes. Under normal conditions, nodes should
// remain in sync through Broadcast messages. For cases
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxShard by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.cluster == nil {
return nil, errors.New("Server.Cluster is nil")
}
if s.holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Node: EncodeNode(s.cluster.Node),
MaxShards: s.holder.encodeMaxShards(),
Schema: s.holder.encodeSchema(),
}
return &ns, nil
}
// HandleRemoteStatus receives incoming NodeStatus from remote nodes.
func (s *Server) HandleRemoteStatus(pb proto.Message) error {
// handleRemoteStatus receives incoming NodeStatus from remote nodes.
func (s *Server) handleRemoteStatus(pb Message) {
// Ignore NodeStatus messages until the cluster is in a Normal state.
if s.cluster.State() != ClusterStateNormal {
return nil
return
}
go func() {
// Make sure the holder has opened.
<-s.holder.opened
err := s.mergeRemoteStatus(pb.(*internal.NodeStatus))
err := s.mergeRemoteStatus(pb.(*NodeStatus))
if err != nil {
s.logger.Printf("merge remote status: %s", err)
}
}()
return nil
}
func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
// Ignore status updates from self.
if s.nodeID == DecodeNode(ns.Node).ID {
if s.nodeID == ns.Node.ID {
return nil
}
@ -612,7 +600,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
// Sync maxShards.
oldmaxshards := s.holder.maxShards()
for index, newMax := range ns.MaxShards.Standard {
for index, newMax := range ns.MaxShards {
localIndex := s.holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to shard creation
@ -641,7 +629,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Host", s.uri.Host)
s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
@ -756,10 +744,3 @@ func expandDirName(path string) (string, error) {
}
return path, nil
}
type MemberServer interface {
ReceiveMessage(proto.Message) error
LocalStatus() (proto.Message, error)
HandleRemoteStatus(proto.Message) error
Node() *Node
}

7
server/config.go
View file

@ -21,13 +21,6 @@ import (
"github.com/pilosa/pilosa/toml"
)
// Cluster types.
const (
ClusterNone = ""
ClusterStatic = "static"
ClusterGossip = "gossip"
)
// TLSConfig contains TLS configuration
type TLSConfig struct {
// CertificatePath contains the path to the certificate (.crt or .pem file)

90
server/handler_test.go
View file

@ -27,10 +27,8 @@ import (
gohttp "net/http"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -62,17 +60,17 @@ func TestHandler_Endpoints(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Shards args protobuf", func(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
Query: "Count(Row(f0=30))",
Shards: []uint64{0, 1},
})
@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 3 {
t.Fatalf("unexpected n: %d", n)
} else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 {
t.Fatalf("unexpected response type: %#v", resp.Results[0])
}
})
@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
} else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
})
t.Run("Row columnattrs protobuf", func(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
Query: "Row(f0=30)",
ColumnAttrs: true,
})
@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
} else if _, ok := resp.Results[0].(*pilosa.Row); !ok {
t.Fatalf("unexpected response type: %#v", resp.Results[0])
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
if a := resp.ColumnAttrSets; len(a) != 2 {
@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if a[0].Attrs["x"] != "y" {
t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"])
}
})
@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
} else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
})
@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: field not found` {
} else if s := resp.Err.Error(); s != `executing: field not found` {
t.Fatalf("unexpected error: %s", s)
}
})
@ -401,7 +393,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Field delete", func(t *testing.T) {
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
if _, err := i.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
@ -453,7 +445,7 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
meta, err := i.CreateFieldIfNotExists("meta", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}

18
server/server.go
View file

@ -35,6 +35,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
@ -139,7 +140,7 @@ func (m *Command) Start() (err error) {
return errors.Wrap(err, "opening server")
}
m.logger.Printf("Listening as %s\n", m.Server.URI)
m.logger.Printf("Listening as %s\n", m.API.Node().URI)
return nil
}
@ -202,7 +203,7 @@ func (m *Command) SetupServer() error {
// Setup TLS
var TLSConfig *tls.Config
if uri.Scheme() == "https" {
if uri.Scheme == "https" {
if m.Config.TLS.CertificatePath == "" {
return errors.New("certificate path is required for TLS sockets")
}
@ -235,7 +236,7 @@ func (m *Command) SetupServer() error {
}
// If port is 0, get auto-allocated port from listener
if uri.Port() == 0 {
if uri.Port == 0 {
uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
}
@ -271,6 +272,7 @@ func (m *Command) SetupServer() error {
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
pilosa.OptServerSerializer(proto.Serializer{}),
coordinatorOpt,
}
@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error {
}
// get the host portion of addr to use for binding
gossipHost := m.Server.URI.Host()
gossipHost := m.API.Node().URI.Host
m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger())
if err != nil {
return errors.Wrap(err, "getting transport")
@ -317,7 +319,7 @@ func (m *Command) SetupNetworking() error {
gossipMemberSet, err := gossip.NewGossipMemberSet(
m.Config.Gossip,
m.Server,
m.API,
gossip.WithLogger(m.logger.Logger()),
gossip.WithTransport(m.gossipTransport),
)
@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
// getListener gets a net.Listener based on the config.
func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
// If bind URI has the https scheme, enable TLS
if uri.Scheme() == "https" && tlsconf != nil {
if uri.Scheme == "https" && tlsconf != nil {
ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf)
if err != nil {
return nil, errors.Wrap(err, "tls.Listener")
}
} else if uri.Scheme() == "http" {
} else if uri.Scheme == "http" {
// Open HTTP listener to determine port (if specified as :0).
ln, err = net.Listen("tcp", uri.HostPort())
if err != nil {
return nil, errors.Wrap(err, "net.Listen")
}
} else {
return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme())
return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme)
}
return ln, nil

2
server/server_test.go
View file

@ -44,7 +44,7 @@ func TestMain_Set_Quick(t *testing.T) {
defer m.Close()
// Create client.
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
t.Fatal(err)
}

12
test/field.go
View file

@ -28,12 +28,12 @@ type Field struct {
}
// NewField returns a new instance of Field d/0.
func NewField(options pilosa.FieldOptions) *Field {
func NewField(opts pilosa.FieldOption) *Field {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
field, err := pilosa.NewField(path, "i", "f", options)
field, err := pilosa.NewField(path, "i", "f", opts)
if err != nil {
panic(err)
}
@ -41,8 +41,8 @@ func NewField(options pilosa.FieldOptions) *Field {
}
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(options pilosa.FieldOptions) *Field {
f := NewField(options)
func MustOpenField(opts pilosa.FieldOption) *Field {
f := NewField(opts)
if err := f.Open(); err != nil {
panic(err)
}
@ -63,7 +63,7 @@ func (f *Field) Reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = pilosa.NewField(path, index, name, pilosa.FieldOptions{})
f.Field, err = pilosa.NewField(path, index, name, pilosa.OptFieldTypeDefault())
if err != nil {
return err
}
@ -76,7 +76,7 @@ func (f *Field) Reopen() error {
// Ensure field can set its cache
func TestField_SetCacheSize(t *testing.T) {
f := MustOpenField(pilosa.FieldOptions{})
f := MustOpenField(pilosa.OptFieldTypeDefault())
defer f.Close()
cacheSize := uint32(100)

19
test/holder.go
View file

@ -81,19 +81,10 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
return &Index{Index: idx}
}
// MustCreateFieldIfNotExists returns a given field. Panic on error.
func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
return f
}
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
@ -106,7 +97,7 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
@ -115,7 +106,7 @@ func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum string) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
@ -129,7 +120,7 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum
// SetBit clears a bit on the given field.
func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
@ -142,7 +133,7 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
// ClearBit clears a bit on the given field.
func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}

8
test/index.go
View file

@ -74,8 +74,8 @@ func (i *Index) Reopen() error {
}
// CreateField creates a field with the given options.
func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateField(name, opt)
func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, error) {
f, err := i.Index.CreateField(name, opts...)
if err != nil {
return nil, err
}
@ -83,8 +83,8 @@ func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error
}
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateFieldIfNotExists(name, opt)
func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) {
f, err := i.Index.CreateFieldIfNotExists(name, opts)
if err != nil {
return nil, err
}

13
test/pilosa.go
View file

@ -63,11 +63,6 @@ func NewCommand(opts ...server.CommandOption) *Command {
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
err = m.SetupServer()
if err != nil {
panic(err)
}
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
@ -117,10 +112,6 @@ func (m *Command) Reopen() error {
config := m.Command.Config
m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...)
m.Command.Config = config
err := m.SetupServer()
if err != nil {
return errors.Wrap(err, "setting up server")
}
// Run new program.
if err := m.Start(); err != nil {
@ -130,11 +121,11 @@ func (m *Command) Reopen() error {
}
// URL returns the base URL string for accessing the running program.
func (m *Command) URL() string { return m.Server.URI.String() }
func (m *Command) URL() string { return m.API.Node().URI.String() }
// Client returns a client to connect to the program.
func (m *Command) Client() *http.InternalClient {
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
panic(err)
}

9
time.go
View file

@ -70,15 +70,6 @@ func (q TimeQuantum) Type() string {
return "TimeQuantum"
}
// ParseTimeQuantum parses v into a time quantum.
func ParseTimeQuantum(v string) (TimeQuantum, error) {
q := TimeQuantum(strings.ToUpper(v))
if !q.Valid() {
return "", ErrInvalidTimeQuantum
}
return q, nil
}
// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
switch unit {

16
time_internal_test.go
View file

@ -16,6 +16,7 @@ package pilosa
import (
"reflect"
"strings"
"testing"
"time"
)
@ -23,7 +24,7 @@ import (
// Ensure string can be parsed into time quantum.
func TestParseTimeQuantum(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if q, err := ParseTimeQuantum("YMDH"); err != nil {
if q, err := parseTimeQuantum("YMDH"); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %#v", q)
@ -31,7 +32,7 @@ func TestParseTimeQuantum(t *testing.T) {
})
t.Run("ErrInvalidTimeQuantum", func(t *testing.T) {
if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
if _, err := parseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
t.Fatalf("unexpected error: %s", err)
}
})
@ -160,9 +161,18 @@ func mustParseTime(value string) time.Time {
// mustParseTimeQuantum parses v into a time quantum. Panic on error.
func mustParseTimeQuantum(v string) TimeQuantum {
q, err := ParseTimeQuantum(v)
q, err := parseTimeQuantum(v)
if err != nil {
panic(err)
}
return q
}
// parseTimeQuantum parses v into a time quantum.
func parseTimeQuantum(v string) (TimeQuantum, error) {
q := TimeQuantum(strings.ToUpper(v))
if !q.Valid() {
return "", ErrInvalidTimeQuantum
}
return q, nil
}

100
uri.go
View file

@ -21,7 +21,6 @@ import (
"strconv"
"strings"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
@ -43,17 +42,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a-
// localhost
// :10101
type URI struct {
scheme string `json:"scheme"`
host string `json:"host"`
port uint16 `json:"port"`
Scheme string `json:"scheme"`
Host string `json:"host"`
Port uint16 `json:"port"`
}
// DefaultURI creates and returns the default URI.
func DefaultURI() *URI {
return &URI{
scheme: "http",
host: "localhost",
port: 10101,
Scheme: "http",
Host: "localhost",
Port: 10101,
}
}
@ -83,44 +82,29 @@ func NewURIFromAddress(address string) (*URI, error) {
return parseAddress(address)
}
// Scheme returns the scheme of this URI.
func (u *URI) Scheme() string {
return u.scheme
}
// SetScheme sets the scheme of this URI.
func (u *URI) SetScheme(scheme string) error {
m := schemeRegexp.FindStringSubmatch(scheme)
if m == nil {
return errors.New("invalid scheme")
}
u.scheme = scheme
u.Scheme = scheme
return nil
}
// Host returns the host of this URI.
func (u *URI) Host() string {
return u.host
}
// SetHost sets the host of this URI.
func (u *URI) SetHost(host string) error {
m := hostRegexp.FindStringSubmatch(host)
if m == nil {
return errors.New("invalid host")
}
u.host = host
u.Host = host
return nil
}
// Port returns the port of this URI.
func (u *URI) Port() uint16 {
return u.port
}
// SetPort sets the port of this URI.
func (u *URI) SetPort(port uint16) {
u.port = port
u.Port = port
}
// HostPort returns `Host:Port`
@ -129,31 +113,23 @@ func (u *URI) HostPort() string {
if u == nil {
return ""
}
s := fmt.Sprintf("%s:%d", u.host, u.port)
s := fmt.Sprintf("%s:%d", u.Host, u.Port)
return s
}
// Normalize returns the address in a form usable by a HTTP client.
func (u *URI) Normalize() string {
scheme := u.scheme
scheme := u.Scheme
index := strings.Index(scheme, "+")
if index >= 0 {
scheme = scheme[:index]
}
return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port)
return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port)
}
// String returns the address as a string.
func (u URI) String() string {
return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port)
}
// Equals returns true if the checked URI is equivalent to this URI.
func (u URI) Equals(other *URI) bool {
if other == nil {
return false
}
return u == *other
return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port)
}
// Path returns URI with path
@ -163,7 +139,7 @@ func (u *URI) Path(path string) string {
// The following methods are required to implement pflag Value interface.
// Set sets the time quantum value.
// Set sets the uri value.
func (u *URI) Set(value string) error {
uri, err := NewURIFromAddress(value)
if err != nil {
@ -173,7 +149,7 @@ func (u *URI) Set(value string) error {
return nil
}
// Type returns the type of a time quantum value.
// Type returns the type of a uri.
func (u URI) Type() string {
return "URI"
}
@ -199,41 +175,13 @@ func parseAddress(address string) (uri *URI, err error) {
}
}
uri = &URI{
scheme: scheme,
host: host,
port: uint16(port),
Scheme: scheme,
Host: host,
Port: uint16(port),
}
return uri, nil
}
// Encode converts o into its internal representation.
func (u URI) Encode() *internal.URI {
return encodeURI(u)
}
func encodeURI(u URI) *internal.URI {
return &internal.URI{
Scheme: u.scheme,
Host: u.host,
Port: uint32(u.port),
}
}
func DecodeURI(i *internal.URI) URI {
return decodeURI(i)
}
func decodeURI(i *internal.URI) URI {
if i == nil {
return URI{}
}
return URI{
scheme: i.Scheme,
host: i.Host,
port: uint16(i.Port),
}
}
// MarshalJSON marshals URI into a JSON-encoded byte slice.
func (u *URI) MarshalJSON() ([]byte, error) {
var output struct {
@ -241,9 +189,9 @@ func (u *URI) MarshalJSON() ([]byte, error) {
Host string `json:"host,omitempty"`
Port uint16 `json:"port,omitempty"`
}
output.Scheme = u.scheme
output.Host = u.host
output.Port = u.port
output.Scheme = u.Scheme
output.Host = u.Host
output.Port = u.Port
return json.Marshal(output)
}
@ -257,8 +205,8 @@ func (u *URI) UnmarshalJSON(b []byte) error {
if err := json.Unmarshal(b, &input); err != nil {
return err
}
u.scheme = input.Scheme
u.host = input.Host
u.port = input.Port
u.Scheme = input.Scheme
u.Host = input.Host
u.Port = input.Port
return nil
}

70
uri_test.go → uri_internal_test.go
View file

@ -1,34 +1,16 @@
// Copyright 2017 Pilosa Corp.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// http://www.apache.org/licenses/LICENSE-2.0
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
@ -94,16 +76,6 @@ func TestURIPath(t *testing.T) {
}
}
func TestEquals(t *testing.T) {
uri1 := DefaultURI()
if uri1.Equals(nil) {
t.Fatalf("URI should not be equal to nil")
}
if !uri1.Equals(DefaultURI()) {
t.Fatalf("URI should be equal to another URI with the same scheme, host and port")
}
}
func TestSetScheme(t *testing.T) {
uri := DefaultURI()
target := "fun"
@ -111,8 +83,8 @@ func TestSetScheme(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if uri.Scheme() != target {
t.Fatalf("%s != %s", uri.Scheme(), target)
if uri.Scheme != target {
t.Fatalf("%s != %s", uri.Scheme, target)
}
}
@ -123,8 +95,8 @@ func TestSetHost(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if uri.Host() != target {
t.Fatalf("%s != %s", uri.host, target)
if uri.Host != target {
t.Fatalf("%s != %s", uri.Host, target)
}
}
@ -132,8 +104,8 @@ func TestSetPort(t *testing.T) {
uri := DefaultURI()
target := uint16(9999)
uri.SetPort(target)
if uri.Port() != target {
t.Fatalf("%d != %d", uri.port, target)
if uri.Port != target {
t.Fatalf("%d != %d", uri.Port, target)
}
}
@ -165,14 +137,14 @@ func TestHostPort(t *testing.T) {
}
func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) {
if uri.Scheme() != scheme {
t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme)
if uri.Scheme != scheme {
t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme)
}
if uri.Host() != host {
t.Fatalf("Host does not match: %s != %s", uri.host, host)
if uri.Host != host {
t.Fatalf("Host does not match: %s != %s", uri.Host, host)
}
if uri.Port() != port {
t.Fatalf("Port does not match: %d != %d", uri.port, port)
if uri.Port != port {
t.Fatalf("Port does not match: %d != %d", uri.Port, port)
}
}

34
utils_internal_test.go
View file

@ -24,7 +24,6 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -104,13 +103,13 @@ func (t *ClusterCluster) CreateIndex(name string) error {
return nil
}
func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error {
func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error {
for _, c := range t.Clusters {
idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateField(field, opt); err != nil {
if _, err := idx.CreateField(field, opts); err != nil {
return err
}
}
@ -162,7 +161,7 @@ func (t *ClusterCluster) addNode() error {
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &nodeEvent{
ev := &NodeEvent{
Event: NodeJoin,
Node: c.Node,
}
@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error {
}
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *ClusterCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
func (t *ClusterCluster) SendSync(m Message) error {
switch obj := m.(type) {
case *ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.mergeClusterStatus(obj)
@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error {
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *ClusterCluster) SendAsync(pb proto.Message) error {
func (t *ClusterCluster) SendAsync(Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
func (t *ClusterCluster) SendTo(to *Node, m Message) error {
switch obj := m.(type) {
case *ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
case *ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.markResizeInstructionComplete(obj)
}
@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error {
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI)
instrNode := DecodeNode(instr.Node)
instrNode := instr.Node
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
@ -365,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
}
for _, src := range instr.Sources {
srcNode := DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcCluster := t.clusterByID(src.Node.ID)
srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard)
@ -405,6 +403,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
complete.Error = err.Error()
}
node := DecodeNode(instr.Coordinator)
node := instr.Coordinator
return t.SendTo(node, complete)
}

9
view.go
View file

@ -22,7 +22,6 @@ import (
"strings"
"sync"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pkg/errors"
)
@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) {
// Send the create shard message to all nodes.
err := v.broadcaster.SendSync(
&internal.CreateShardMessage{
&CreateShardMessage{
Index: v.index,
Shard: shard,
})
@ -422,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*
return r, nil
}
// viewInfo represents schema information for a view.
type viewInfo struct {
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`
}
type viewInfoSlice []*viewInfo
type viewInfoSlice []*ViewInfo
func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int { return len(p) }