diff --git a/api.go b/api.go index 33fff083d..8fe028675 100644 --- a/api.go +++ b/api.go @@ -26,8 +26,6 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -38,6 +36,8 @@ type API struct { holder *Holder cluster *cluster server *Server + + Serializer Serializer } // APIOption is a functional option type for pilosa.API @@ -48,6 +48,7 @@ func OptAPIServer(s *Server) APIOption { a.server = s a.holder = s.holder a.cluster = s.cluster + a.Serializer = s.serializer return nil } } @@ -149,6 +150,10 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, nil } +func (api *API) Holder() *Holder { + return api.server.Holder() +} + // readColumnAttrSets returns a list of column attribute objects by id. func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { if index == nil { @@ -185,12 +190,11 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index } // Send the create index message to all nodes. err = api.server.SendSync( - &internal.CreateIndexMessage{ + &CreateIndexMessage{ Index: indexName, - Meta: options.Encode(), + Meta: &options, }) if err != nil { - api.server.logger.Printf("problem sending CreateIndex message: %s", err) return nil, errors.Wrap(err, "sending CreateIndex message") } api.holder.Stats.Count("createIndex", 1, 1.0) @@ -224,7 +228,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // Send the delete index message to all nodes. err = api.server.SendSync( - &internal.DeleteIndexMessage{ + &DeleteIndexMessage{ Index: indexName, }) if err != nil { @@ -238,16 +242,18 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateField makes the named field in the named index with the given options. // This method currently only takes a single functional option, but that may be // changed in the future to support multiple options. -func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts FieldOption) (*Field, error) { +func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { if err := api.validate(apiCreateField); err != nil { return nil, errors.Wrap(err, "validating api method") } - // Apply functional option. + // Apply functional options. fo := FieldOptions{} - err := opts(&fo) - if err != nil { - return nil, errors.Wrap(err, "applying option") + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } } // Find index. @@ -257,17 +263,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str } // Create field. - field, err := index.CreateField(fieldName, fo) + field, err := index.CreateField(fieldName, opts...) if err != nil { return nil, errors.Wrap(err, "creating field") } // Send the create field message to all nodes. err = api.server.SendSync( - &internal.CreateFieldMessage{ + &CreateFieldMessage{ Index: indexName, Field: fieldName, - Meta: fo.Encode(), + Meta: &fo, }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) @@ -311,7 +317,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str // Send the delete field message to all nodes. err := api.server.SendSync( - &internal.DeleteFieldMessage{ + &DeleteFieldMessage{ Index: indexName, Field: fieldName, }) @@ -331,8 +337,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } // Validate that this handler owns the shard. - if !api.cluster.ownsShard(api.LocalID(), indexName, shard) { - api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) return ErrClusterDoesNotOwnShard } @@ -370,55 +376,6 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) return api.cluster.shardNodes(indexName, shard), nil } -// MarshalFragment returns an object which can write the specified fragment's data -// to an io.Writer. The serialized data can be read back into a fragment with -// the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) { - if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - // Retrieve fragment from holder. - f := api.holder.fragment(indexName, fieldName, viewStandard, shard) - if f == nil { - return nil, ErrFragmentNotFound - } - return f, nil -} - -// UnmarshalFragment creates a new fragment (if necessary) and reads data from a -// Reader which was previously written by MarshalFragment to populate the -// fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error { - if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve field. - f := api.holder.Field(indexName, fieldName) - if f == nil { - return ErrFieldNotFound - } - - // Retrieve view. - view, err := f.createViewIfNotExists(viewStandard) - if err != nil { - return errors.Wrap(err, "creating view") - } - - // Retrieve fragment from field. - frag, err := view.CreateFragmentIfNotExists(shard) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - // Read fragment in from request body. - if _, err := frag.ReadFrom(reader); err != nil { - return errors.Wrap(err, "reading fragment") - } - return nil -} - // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. @@ -431,8 +388,8 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read body error")) } - var req internal.BlockDataRequest - if err := proto.Unmarshal(reqBytes, &req); err != nil { + var req BlockDataRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) } @@ -442,11 +399,11 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, return nil, ErrFragmentNotFound } - var resp = internal.BlockDataResponse{} + var resp = BlockDataResponse{} resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. - buf, err := proto.Marshal(&resp) + buf, err := api.Serializer.Marshal(&resp) if err != nil { return nil, errors.Wrap(err, "merge block response encoding error") } @@ -477,13 +434,19 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.cluster.Nodes } +// Node gets the ID, URI and coordinator status for this particular node. +func (api *API) Node() *Node { + node := api.server.node() + return &node +} + // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { return errors.Wrap(err, "validating api method") } - err := api.server.SendSync(&internal.RecalculateCaches{}) + err := api.server.SendSync(&RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") } @@ -504,28 +467,24 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { return errors.Wrap(err, "reading body") } - // Marshal into request object. - pb, err := UnmarshalMessage(body) + typ := body[0] + msg := getMessage(typ) + err = api.server.serializer.Unmarshal(body[1:], msg) if err != nil { - return errors.Wrap(err, "unmarshaling message") + return errors.Wrap(err, "deserializing cluster message") } // Forward the error message. - if err := api.server.ReceiveMessage(pb); err != nil { + if err := api.server.receiveMessage(msg); err != nil { return errors.Wrap(err, "receiving message") } return nil } -// LocalID returns the current node's ID. -func (api *API) LocalID() string { - return api.cluster.Node.ID -} - // Schema returns information about each index in Pilosa including which fields -// and views they contain. +// they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { - return api.holder.Schema() + return api.holder.limitedSchema() } // Views returns the views in the given field. @@ -567,7 +526,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri // Send the delete view message to all nodes. err := api.server.SendSync( - &internal.DeleteViewMessage{ + &DeleteViewMessage{ Index: indexName, Field: fieldName, View: viewName, @@ -649,7 +608,7 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s } // Import bulk imports data into a particular index,field,shard. -func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { +func (api *API) Import(ctx context.Context, req *ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } @@ -678,7 +637,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { +func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { return errors.Wrap(err, "validating api method") } @@ -720,8 +679,8 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) { // Validate that this handler owns the shard. - if !api.cluster.ownsShard(api.LocalID(), indexName, shard) { - api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) return nil, nil, ErrClusterDoesNotOwnShard } @@ -755,15 +714,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode } // If the new coordinator is this node, do the SetCoordinator directly. - if newNode.ID == api.LocalID() { + if newNode.ID == api.Node().ID { return oldNode, newNode, api.cluster.setCoordinator(newNode) } // Send the set-coordinator message to new node. err = api.server.SendTo( newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), + &SetCoordinatorMessage{ + New: newNode, }) if err != nil { return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) @@ -888,7 +847,6 @@ const ( apiIndexAttrDiff //apiLocalID // not implemented //apiLongQueryTime // not implemented - apiMarshalFragment //apiMaxShards // not implemented apiQuery apiRecalculateCaches @@ -899,15 +857,13 @@ const ( apiShardNodes //apiState // not implemented //apiStatsWithTags // not implemented - apiUnmarshalFragment //apiVersion // not implemented apiViews ) var methodsCommon = map[apiMethod]struct{}{ - apiClusterMessage: struct{}{}, - apiMarshalFragment: struct{}{}, - apiSetCoordinator: struct{}{}, + apiClusterMessage: struct{}{}, + apiSetCoordinator: struct{}{}, } var methodsResizing = map[apiMethod]struct{}{ @@ -933,6 +889,5 @@ var methodsNormal = map[apiMethod]struct{}{ apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, apiShardNodes: struct{}{}, - apiUnmarshalFragment: struct{}{}, apiViews: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index 881b79472..01217092f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 224, 232, 252, 265, 279, 296, 309, 329, 337} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/broadcast.go b/broadcast.go index 37452292a..a3ea01a4f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -16,20 +16,27 @@ package pilosa import ( "fmt" - "reflect" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) +// Serializer is an interface for serializing pilosa types to bytes and back. +type Serializer interface { + Marshal(Message) ([]byte, error) + Unmarshal([]byte, Message) error +} + // broadcaster is an interface for broadcasting messages. type broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error - SendTo(to *Node, pb proto.Message) error + SendSync(Message) error + SendAsync(Message) error + SendTo(*Node, Message) error } +// Message is the interface implemented by all core pilosa types which can be serialized to messages. +// TODO add at least a single "isMessage()" method. +type Message interface{} + func init() { NopBroadcaster = &nopBroadcaster{} } @@ -40,13 +47,13 @@ var NopBroadcaster broadcaster type nopBroadcaster struct{} // SendSync A no-op implementation of Broadcaster SendSync method. -func (n nopBroadcaster) SendSync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendSync(Message) error { return nil } // SendAsync A no-op implementation of Broadcaster SendAsync method. -func (n nopBroadcaster) SendAsync(pb proto.Message) error { return nil } +func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (c nopBroadcaster) SendTo(to *Node, pb proto.Message) error { return nil } +func (nopBroadcaster) SendTo(*Node, Message) error { return nil } // Broadcast message types. const ( @@ -65,94 +72,94 @@ const ( messageTypeNodeState messageTypeRecalculateCaches messageTypeNodeEvent + messageTypeNodeStatus ) -// MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateShardMessage: - typ = messageTypeCreateShard - case *internal.CreateIndexMessage: - typ = messageTypeCreateIndex - case *internal.DeleteIndexMessage: - typ = messageTypeDeleteIndex - case *internal.CreateFieldMessage: - typ = messageTypeCreateField - case *internal.DeleteFieldMessage: - typ = messageTypeDeleteField - case *internal.CreateViewMessage: - typ = messageTypeCreateView - case *internal.DeleteViewMessage: - typ = messageTypeDeleteView - case *internal.ClusterStatus: - typ = messageTypeClusterStatus - case *internal.ResizeInstruction: - typ = messageTypeResizeInstruction - case *internal.ResizeInstructionComplete: - typ = messageTypeResizeInstructionComplete - case *internal.SetCoordinatorMessage: - typ = messageTypeSetCoordinator - case *internal.UpdateCoordinatorMessage: - typ = messageTypeUpdateCoordinator - case *internal.NodeStateMessage: - typ = messageTypeNodeState - case *internal.RecalculateCaches: - typ = messageTypeRecalculateCaches - case *internal.NodeEventMessage: - typ = messageTypeNodeEvent - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) - } - buf, err := proto.Marshal(m) +// MarshalInternalMessage serializes the pilosa message and adds pilosa internal +// type info which is used by the internal messaging stuff. +func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) { + typ := getMessageType(m) + buf, err := s.Marshal(m) if err != nil { - return nil, errors.Wrap(err, "marshalling") + return nil, errors.Wrap(err, "marshaling") } return append([]byte{typ}, buf...), nil } -// UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - - var m proto.Message +func getMessage(typ byte) Message { switch typ { case messageTypeCreateShard: - m = &internal.CreateShardMessage{} + return &CreateShardMessage{} case messageTypeCreateIndex: - m = &internal.CreateIndexMessage{} + return &CreateIndexMessage{} case messageTypeDeleteIndex: - m = &internal.DeleteIndexMessage{} + return &DeleteIndexMessage{} case messageTypeCreateField: - m = &internal.CreateFieldMessage{} + return &CreateFieldMessage{} case messageTypeDeleteField: - m = &internal.DeleteFieldMessage{} + return &DeleteFieldMessage{} case messageTypeCreateView: - m = &internal.CreateViewMessage{} + return &CreateViewMessage{} case messageTypeDeleteView: - m = &internal.DeleteViewMessage{} + return &DeleteViewMessage{} case messageTypeClusterStatus: - m = &internal.ClusterStatus{} + return &ClusterStatus{} case messageTypeResizeInstruction: - m = &internal.ResizeInstruction{} + return &ResizeInstruction{} case messageTypeResizeInstructionComplete: - m = &internal.ResizeInstructionComplete{} + return &ResizeInstructionComplete{} case messageTypeSetCoordinator: - m = &internal.SetCoordinatorMessage{} + return &SetCoordinatorMessage{} case messageTypeUpdateCoordinator: - m = &internal.UpdateCoordinatorMessage{} + return &UpdateCoordinatorMessage{} case messageTypeNodeState: - m = &internal.NodeStateMessage{} + return &NodeStateMessage{} case messageTypeRecalculateCaches: - m = &internal.RecalculateCaches{} + return &RecalculateCaches{} case messageTypeNodeEvent: - m = &internal.NodeEventMessage{} + return &NodeEvent{} + case messageTypeNodeStatus: + return &NodeStatus{} default: - return nil, fmt.Errorf("invalid message type: %d", typ) + panic(fmt.Sprintf("unknown message type %d", typ)) + } +} + +func getMessageType(m Message) byte { + switch m.(type) { + case *CreateShardMessage: + return messageTypeCreateShard + case *CreateIndexMessage: + return messageTypeCreateIndex + case *DeleteIndexMessage: + return messageTypeDeleteIndex + case *CreateFieldMessage: + return messageTypeCreateField + case *DeleteFieldMessage: + return messageTypeDeleteField + case *CreateViewMessage: + return messageTypeCreateView + case *DeleteViewMessage: + return messageTypeDeleteView + case *ClusterStatus: + return messageTypeClusterStatus + case *ResizeInstruction: + return messageTypeResizeInstruction + case *ResizeInstructionComplete: + return messageTypeResizeInstructionComplete + case *SetCoordinatorMessage: + return messageTypeSetCoordinator + case *UpdateCoordinatorMessage: + return messageTypeUpdateCoordinator + case *NodeStateMessage: + return messageTypeNodeState + case *RecalculateCaches: + return messageTypeRecalculateCaches + case *NodeEvent: + return messageTypeNodeEvent + case *NodeStatus: + return messageTypeNodeStatus + default: + panic(fmt.Sprintf("don't have type for message %#v", m)) } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - return m, nil } diff --git a/broadcast_test.go b/broadcast_test.go deleted file mode 100644 index 415228718..000000000 --- a/broadcast_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "reflect" - "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" -) - -// Ensure a message can be marshaled and unmarshaled. -func TestMessage_Marshal(t *testing.T) { - - testMessageMarshal(t, &internal.CreateShardMessage{ - Index: "i", - Shard: 8, - }) - - testMessageMarshal(t, &internal.DeleteIndexMessage{ - Index: "i", - }) -} - -func testMessageMarshal(t *testing.T, m proto.Message) { - marshalled, err := pilosa.MarshalMessage(m) - if err != nil { - t.Fatal(err) - } - unmarshalled, err := pilosa.UnmarshalMessage(marshalled) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(unmarshalled, m) { - t.Fatalf("unexpected message marshalling: %s", unmarshalled) - } -} diff --git a/cache.go b/cache.go index 192f38cc7..044d85f3c 100644 --- a/cache.go +++ b/cache.go @@ -22,7 +22,6 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/lru" ) @@ -318,22 +317,6 @@ type Pair struct { Count uint64 `json:"count"` } -func encodePair(p Pair) *internal.Pair { - return &internal.Pair{ - ID: p.ID, - Key: p.Key, - Count: p.Count, - } -} - -func decodePair(pb *internal.Pair) Pair { - return Pair{ - ID: pb.ID, - Key: pb.Key, - Count: pb.Count, - } -} - // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -409,22 +392,6 @@ func (p Pairs) String() string { return buf.String() } -func EncodePairs(a Pairs) []*internal.Pair { - other := make([]*internal.Pair, len(a)) - for i := range a { - other[i] = encodePair(a[i]) - } - return other -} - -func decodePairs(a []*internal.Pair) []Pair { - other := make([]Pair, len(a)) - for i := range a { - other[i] = decodePair(a[i]) - } - return other -} - // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/client.go b/client.go index 59e3ad59b..5c51ae63f 100644 --- a/client.go +++ b/client.go @@ -3,9 +3,6 @@ package pilosa import ( "context" "io" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // Bit represents the intersection of a row and a column. It can be specifed by @@ -36,8 +33,8 @@ type InternalClient interface { Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) - Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error @@ -49,19 +46,19 @@ type InternalClient interface { BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) } //=============== type InternalQueryClient interface { - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) } type NopInternalQueryClient struct{} -func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } @@ -91,10 +88,10 @@ func (n NopInternalClient) CreateIndex(ctx context.Context, index string, opt In func (n NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } -func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } -func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (n NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } func (n NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { @@ -128,7 +125,7 @@ func (n NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index s func (n NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { return nil, nil } -func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { +func (n NopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } func (n NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { diff --git a/cluster.go b/cluster.go index acf06dae0..f61173c3b 100644 --- a/cluster.go +++ b/cluster.go @@ -45,8 +45,7 @@ const ( ClusterStateResizing = "RESIZING" // NodeState represents the state of a node during startup. - NodeStateLoading = "LOADING" - NodeStateReady = "READY" + NodeStateReady = "READY" // resizeJob states. resizeJobStateRunning = "RUNNING" @@ -69,52 +68,6 @@ func (n Node) String() string { return fmt.Sprintf("Node: %s", n.ID) } -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *nodeEvent { - return &nodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - // Nodes represents a list of nodes. type Nodes []*Node @@ -314,8 +267,8 @@ func (c *cluster) setCoordinator(n *Node) error { c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.broadcaster.SendSync( - &internal.UpdateCoordinatorMessage{ - New: EncodeNode(n), + &UpdateCoordinatorMessage{ + New: n, }) if err != nil { return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) @@ -469,7 +422,7 @@ func (c *cluster) setNodeState(state string) error { } // Send node state to coordinator. - ns := &internal.NodeStateMessage{ + ns := &NodeStateMessage{ NodeID: c.Node.ID, State: state, } @@ -506,12 +459,12 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error { return nil } -// Status returns the internal ClusterStatus representation. -func (c *cluster) Status() *internal.ClusterStatus { - return &internal.ClusterStatus{ +// Status returns the the cluster's status including what nodes it contains, its ID, and current state. +func (c *cluster) Status() *ClusterStatus { + return &ClusterStatus{ ClusterID: c.id, State: c.state, - Nodes: EncodeNodes(c.Nodes), + Nodes: c.Nodes, } } @@ -686,8 +639,8 @@ func (c *cluster) diff(other *cluster) (action string, nodeID string, err error) // fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { - m := make(map[string][]*internal.ResizeSource) +func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) { + m := make(map[string][]*ResizeSource) // Determine if a node is being added or removed. action, diffNodeID, err := c.diff(to) @@ -746,7 +699,7 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R // Get the ResizeSource for each diff. for nodeID, diff := range diffs { - m[nodeID] = []*internal.ResizeSource{} + m[nodeID] = []*ResizeSource{} for _, frag := range diff { // If there is no valid source node ID for a fragment, // it likely means that the replica factor was not @@ -757,8 +710,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*internal.R return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") } - src := &internal.ResizeSource{ - Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), + src := &ResizeSource{ + Node: c.unprotectedNodeByID(srcNodeID), Index: idx.Name(), Field: frag.field, View: frag.view, @@ -902,9 +855,9 @@ func (c *cluster) waitForStarted() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &internal.NodeEventMessage{ - Event: uint32(NodeJoin), - Node: EncodeNode(c.Node), + msg := &NodeEvent{ + Event: NodeJoin, + Node: c.Node, } if err := c.broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) @@ -1014,8 +967,8 @@ func (c *cluster) setStateAndBroadcast(state string) error { return c.broadcaster.SendSync(c.Status()) } -func (c *cluster) sendTo(node *Node, msg proto.Message) error { - if err := c.broadcaster.SendTo(node, msg); err != nil { +func (c *cluster) sendTo(node *Node, m Message) error { + if err := c.broadcaster.SendTo(node, m); err != nil { return errors.Wrap(err, "sending") } return nil @@ -1121,7 +1074,7 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, } // multiIndex is a map of sources initialized with all the nodes in toCluster. - multiIndex := make(map[string][]*internal.ResizeSource) + multiIndex := make(map[string][]*ResizeSource) for _, n := range toCluster.Nodes { multiIndex[n.ID] = nil @@ -1145,12 +1098,12 @@ func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, j.IDs[id] = true continue } - instr := &internal.ResizeInstruction{ + instr := &ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.unprotectedNodeByID(id)), - Coordinator: EncodeNode(c.coordinatorNode()), + Node: toCluster.unprotectedNodeByID(id), + Coordinator: c.coordinatorNode(), Sources: sources, - Schema: c.holder.encodeSchema(), // Include the schema to ensure it's in sync on the receiving node. + Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) @@ -1176,7 +1129,7 @@ func (c *cluster) completeCurrentJob(state string) error { } // followResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { +func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { c.logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. @@ -1194,7 +1147,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err <-c.holder.opened // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -1213,7 +1166,7 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err for _, src := range instr.Sources { c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - srcURI := decodeURI(src.Node.URI) + srcURI := src.Node.URI // Retrieve field. f := c.holder.Field(src.Index, src.Field) @@ -1265,14 +1218,14 @@ func (c *cluster) followResizeInstruction(instr *internal.ResizeInstruction) err complete.Error = err.Error() } - if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { + if err := c.sendTo(instr.Coordinator, complete); err != nil { c.logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil } -func (c *cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error { j := c.job(complete.JobID) @@ -1309,7 +1262,7 @@ func (c *cluster) job(id int64) *resizeJob { type resizeJob struct { ID int64 IDs map[string]bool - Instructions []*internal.ResizeInstruction + Instructions []*ResizeInstruction Broadcaster broadcaster action string @@ -1412,7 +1365,7 @@ func (j *resizeJob) distributeResizeInstructions() error { // a dummy node object to use in the SendTo() method. node := &Node{ ID: instr.Node.ID, - URI: decodeURI(instr.Node.URI), + URI: instr.Node.URI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1553,32 +1506,6 @@ func (c *cluster) saveTopology() error { return nil } -func encodeTopology(topology *Topology) *internal.Topology { - if topology == nil { - return nil - } - return &internal.Topology{ - ClusterID: topology.ClusterID, - NodeIDs: topology.NodeIDs, - } -} - -func decodeTopology(topology *internal.Topology) (*Topology, error) { - if topology == nil { - return nil, nil - } - - t := NewTopology() - t.ClusterID = topology.ClusterID - t.NodeIDs = topology.NodeIDs - sort.Slice(t.NodeIDs, - func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] - }) - - return t, nil -} - func (c *cluster) considerTopology() error { // Create ClusterID if one does not already exist. if c.id == "" { @@ -1612,7 +1539,7 @@ func (c *cluster) considerTopology() error { } // ReceiveEvent represents an implementation of EventHandler. -func (c *cluster) ReceiveEvent(e *nodeEvent) error { +func (c *cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. if e.Node.ID == c.Node.ID { return nil @@ -1752,7 +1679,7 @@ func (c *cluster) nodeLeave(node *Node) error { return nil } -func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.logger.Printf("merge cluster status: %v", cs) @@ -1764,7 +1691,7 @@ func (c *cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { // Set ClusterID. c.setID(cs.ClusterID) - officialNodes := DecodeNodes(cs.Nodes) + officialNodes := cs.Nodes // Add all nodes from the coordinator. for _, node := range officialNodes { @@ -1813,3 +1740,120 @@ func (c *cluster) setStatic(hosts []string) error { } return nil } + +type ClusterStatus struct { + ClusterID string + State string + Nodes []*Node +} + +type ResizeInstruction struct { + JobID int64 + Node *Node + Coordinator *Node + Sources []*ResizeSource + Schema *Schema + ClusterStatus *ClusterStatus +} + +type ResizeSource struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` +} + +// Schema contains information about indexes and their configuration. +type Schema struct { + Indexes []*IndexInfo +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + ClusterID: topology.ClusterID, + NodeIDs: topology.NodeIDs, + } +} + +func decodeTopology(topology *internal.Topology) (*Topology, error) { + if topology == nil { + return nil, nil + } + + t := NewTopology() + t.ClusterID = topology.ClusterID + t.NodeIDs = topology.NodeIDs + sort.Slice(t.NodeIDs, + func(i, j int) bool { + return t.NodeIDs[i] < t.NodeIDs[j] + }) + + return t, nil +} + +type CreateShardMessage struct { + Index string + Shard uint64 +} + +type CreateIndexMessage struct { + Index string + Meta *IndexOptions +} + +type DeleteIndexMessage struct { + Index string +} + +type CreateFieldMessage struct { + Index string + Field string + Meta *FieldOptions +} + +type DeleteFieldMessage struct { + Index string + Field string +} + +type CreateViewMessage struct { + Index string + Field string + View string +} +type DeleteViewMessage struct { + Index string + Field string + View string +} + +type ResizeInstructionComplete struct { + JobID int64 + Node *Node + Error string +} + +type SetCoordinatorMessage struct { + New *Node +} + +type UpdateCoordinatorMessage struct { + New *Node +} + +type NodeStateMessage struct { + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +type NodeStatus struct { + Node *Node + MaxShards map[string]uint64 + Schema *Schema +} + +type RecalculateCaches struct{} diff --git a/cluster_internal_test.go b/cluster_internal_test.go index aee4ea09b..06e7dd3d7 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -24,7 +24,6 @@ import ( "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -150,7 +149,7 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - field, err := idx.CreateFieldIfNotExists("f", FieldOptions{}) + field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) if err != nil { t.Fatal(err) } @@ -175,19 +174,19 @@ func TestFragSources(t *testing.T) { from *cluster to *cluster idx *Index - expected map[string][]*internal.ResizeSource + expected map[string][]*ResizeSource err string }{ { from: c1, to: c2, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{}, - "node1": []*internal.ResizeSource{}, - "node2": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{}, + "node1": []*ResizeSource{}, + "node2": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -196,13 +195,13 @@ func TestFragSources(t *testing.T) { from: c4, to: c3, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -211,15 +210,15 @@ func TestFragSources(t *testing.T) { from: c5, to: c4, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, }, - "node2": []*internal.ResizeSource{}, + "node2": []*ResizeSource{}, }, err: "", }, @@ -697,7 +696,7 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { + if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil { t.Fatal(err) } tc.SetBit("i", "f", 1, 101, nil) diff --git a/ctl/export_test.go b/ctl/export_test.go index e959da6bc..e3189efe3 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -49,7 +49,7 @@ func TestExportCommand_Run(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewExportCommand(stdin, stdout, stderr) - hostport := cmd.Server.URI.HostPort() + hostport := cmd.API.Node().URI.HostPort() cm.Host = hostport http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) diff --git a/ctl/import_test.go b/ctl/import_test.go index 32792c02d..56970e9be 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -62,7 +62,7 @@ func TestImportCommand_Run(t *testing.T) { } cmd := test.MustRunCluster(t, 1)[0] - cm.Host = cmd.Server.URI.HostPort() + cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" cm.Field = "f" @@ -87,7 +87,7 @@ func TestImportCommand_RunValue(t *testing.T) { } cmd := test.MustRunCluster(t, 1)[0] - cm.Host = cmd.Server.URI.HostPort() + cm.Host = cmd.API.Node().URI.HostPort() http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) @@ -107,7 +107,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = cmd.Server.URI.HostPort() + cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") @@ -188,7 +188,7 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { t.Fatal(err) } - cm.Host = cmd.Server.URI.HostPort() + cm.Host = cmd.API.Node().URI.HostPort() http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go new file mode 100644 index 000000000..0020260ab --- /dev/null +++ b/encoding/proto/proto.go @@ -0,0 +1,1037 @@ +package proto + +import ( + "fmt" + "sort" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// Serializer implements pilosa.Serializer for protobufs. +type Serializer struct{} + +// Marshal turns pilosa messages into protobuf serialized bytes. +func (Serializer) Marshal(m pilosa.Message) ([]byte, error) { + pm := encodeToProto(m) + if pm == nil { + return nil, errors.New("passed invalid pilosa.Message") + } + buf, err := proto.Marshal(pm) + return buf, errors.Wrap(err, "marshalling") +} + +// Unmarshal takes byte slices and protobuf deserializes them into a pilosa Message. +func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + msg := &internal.CreateShardMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateShardMessage") + } + decodeCreateShardMessage(msg, mt) + return nil + case *pilosa.CreateIndexMessage: + msg := &internal.CreateIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateIndexMessage") + } + decodeCreateIndexMessage(msg, mt) + return nil + case *pilosa.DeleteIndexMessage: + msg := &internal.DeleteIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteIndexMessage") + } + decodeDeleteIndexMessage(msg, mt) + return nil + case *pilosa.CreateFieldMessage: + msg := &internal.CreateFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateFieldMessage") + } + decodeCreateFieldMessage(msg, mt) + return nil + case *pilosa.DeleteFieldMessage: + msg := &internal.DeleteFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteFieldMessage") + } + decodeDeleteFieldMessage(msg, mt) + return nil + case *pilosa.CreateViewMessage: + msg := &internal.CreateViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateViewMessage") + } + decodeCreateViewMessage(msg, mt) + return nil + case *pilosa.DeleteViewMessage: + msg := &internal.DeleteViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteViewMessage") + } + decodeDeleteViewMessage(msg, mt) + return nil + case *pilosa.ClusterStatus: + msg := &internal.ClusterStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ClusterStatus") + } + decodeClusterStatus(msg, mt) + return nil + case *pilosa.ResizeInstruction: + msg := &internal.ResizeInstruction{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstruction") + } + decodeResizeInstruction(msg, mt) + return nil + case *pilosa.ResizeInstructionComplete: + msg := &internal.ResizeInstructionComplete{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstructionComplete") + } + decodeResizeInstructionComplete(msg, mt) + return nil + case *pilosa.SetCoordinatorMessage: + msg := &internal.SetCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling SetCoordinatorMessage") + } + decodeSetCoordinatorMessage(msg, mt) + return nil + case *pilosa.UpdateCoordinatorMessage: + msg := &internal.UpdateCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage") + } + decodeUpdateCoordinatorMessage(msg, mt) + return nil + case *pilosa.NodeStateMessage: + msg := &internal.NodeStateMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStateMessage") + } + decodeNodeStateMessage(msg, mt) + return nil + case *pilosa.RecalculateCaches: + msg := &internal.RecalculateCaches{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling RecalculateCaches") + } + decodeRecalculateCaches(msg, mt) + return nil + case *pilosa.NodeEvent: + msg := &internal.NodeEventMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeEvent") + } + decodeNodeEventMessage(msg, mt) + return nil + case *pilosa.NodeStatus: + msg := &internal.NodeStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStatus") + } + decodeNodeStatus(msg, mt) + return nil + case *pilosa.Node: + msg := &internal.Node{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling Node") + } + decodeNode(msg, mt) + return nil + case *pilosa.QueryRequest: + msg := &internal.QueryRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryRequest") + } + decodeQueryRequest(msg, mt) + return nil + case *pilosa.QueryResponse: + msg := &internal.QueryResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryResponse") + } + decodeQueryResponse(msg, mt) + return nil + case *pilosa.ImportRequest: + msg := &internal.ImportRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRequest") + } + decodeImportRequest(msg, mt) + return nil + case *pilosa.ImportValueRequest: + msg := &internal.ImportValueRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportValueRequest") + } + decodeImportValueRequest(msg, mt) + return nil + case *pilosa.ImportResponse: + msg := &internal.ImportResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportResponse") + } + decodeImportResponse(msg, mt) + return nil + case *pilosa.BlockDataRequest: + msg := &internal.BlockDataRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataRequest") + } + decodeBlockDataRequest(msg, mt) + return nil + case *pilosa.BlockDataResponse: + msg := &internal.BlockDataResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataResponse") + } + decodeBlockDataResponse(msg, mt) + return nil + default: + panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) + } +} + +func encodeToProto(m pilosa.Message) proto.Message { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + return encodeCreateShardMessage(mt) + case *pilosa.CreateIndexMessage: + return encodeCreateIndexMessage(mt) + case *pilosa.DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *pilosa.CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *pilosa.DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *pilosa.CreateViewMessage: + return encodeCreateViewMessage(mt) + case *pilosa.DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *pilosa.ClusterStatus: + return encodeClusterStatus(mt) + case *pilosa.ResizeInstruction: + return encodeResizeInstruction(mt) + case *pilosa.ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *pilosa.SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *pilosa.UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *pilosa.NodeStateMessage: + return encodeNodeStateMessage(mt) + case *pilosa.RecalculateCaches: + return encodeRecalculateCaches(mt) + case *pilosa.NodeEvent: + return encodeNodeEventMessage(mt) + case *pilosa.NodeStatus: + return encodeNodeStatus(mt) + case *pilosa.Node: + return encodeNode(mt) + case *pilosa.QueryRequest: + return encodeQueryRequest(mt) + case *pilosa.QueryResponse: + return encodeQueryResponse(mt) + case *pilosa.ImportRequest: + return encodeImportRequest(mt) + case *pilosa.ImportValueRequest: + return encodeImportValueRequest(mt) + case *pilosa.ImportResponse: + return encodeImportResponse(mt) + case *pilosa.BlockDataRequest: + return encodeBlockDataRequest(mt) + case *pilosa.BlockDataResponse: + return encodeBlockDataResponse(mt) + } + return nil +} + +func encodeBlockDataRequest(m *pilosa.BlockDataRequest) *internal.BlockDataRequest { + return &internal.BlockDataRequest{ + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + Block: m.Block, + } +} +func encodeBlockDataResponse(m *pilosa.BlockDataResponse) *internal.BlockDataResponse { + return &internal.BlockDataResponse{ + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + } +} + +func encodeImportResponse(m *pilosa.ImportResponse) *internal.ImportResponse { + return &internal.ImportResponse{ + Err: m.Err, + } +} + +func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { + return &internal.ImportRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, + } +} + +func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { + return &internal.ImportValueRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + } +} + +func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { + return &internal.QueryRequest{ + Query: m.Query, + Shards: m.Shards, + ColumnAttrs: m.ColumnAttrs, + Remote: m.Remote, + ExcludeRowAttrs: m.ExcludeRowAttrs, + ExcludeColumns: m.ExcludeColumns, + } +} + +func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(m.Results)), + ColumnAttrSets: EncodeColumnAttrSets(m.ColumnAttrSets), + } + + for i := range m.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := m.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = queryResultTypeRow + pb.Results[i].Row = EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = queryResultTypePairs + pb.Results[i].Pairs = EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = queryResultTypeValCount + pb.Results[i].ValCount = EncodeValCount(result) + case uint64: + pb.Results[i].Type = queryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = queryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = queryResultTypeNil + } + } + + if m.Err != nil { + pb.Err = m.Err.Error() + } + + return pb +} + +func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: encodeNode(m.Node), + Coordinator: encodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + +func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + +func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: encodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + +func encodeSchema(m *pilosa.Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + +func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + +func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + +func encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + +func encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + +func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +// EncodeNodes converts a slice of Nodes into its internal representation. +func EncodeNodes(a []*pilosa.Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = encodeNode(a[i]) + } + return other +} + +// encodeNode converts a Node into its internal representation. +func encodeNode(n *pilosa.Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: encodeURI(n.URI), + IsCoordinator: n.IsCoordinator, + } +} + +func encodeURI(u pilosa.URI) *internal.URI { + return &internal.URI{ + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), + } +} + +func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: EncodeNodes(m.Nodes), + } +} + +func encodeCreateShardMessage(m *pilosa.CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: encodeNode(m.Node), + Error: m.Error, + } +} + +func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: encodeNode(m.New), + } +} + +func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: encodeNode(m.New), + } +} + +func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: encodeNode(m.Node), + } +} + +func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: encodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { + m.JobID = ri.JobID + m.Node = &pilosa.Node{} + decodeNode(ri.Node, m.Node) + m.Coordinator = &pilosa.Node{} + decodeNode(ri.Coordinator, m.Coordinator) + m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) + decodeResizeSources(ri.Sources, m.Sources) + m.Schema = &pilosa.Schema{} + decodeSchema(ri.Schema, m.Schema) + m.ClusterStatus = &pilosa.ClusterStatus{} + decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) +} + +func decodeResizeSources(srcs []*internal.ResizeSource, m []*pilosa.ResizeSource) { + for i := range srcs { + m[i] = &pilosa.ResizeSource{} + decodeResizeSource(srcs[i], m[i]) + } +} + +func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) { + m.Node = &pilosa.Node{} + decodeNode(rs.Node, m.Node) + m.Index = rs.Index + m.Field = rs.Field + m.View = rs.View + m.Shard = rs.Shard +} + +func decodeSchema(s *internal.Schema, m *pilosa.Schema) { + m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes)) + decodeIndexes(s.Indexes, m.Indexes) +} + +func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) { + for i := range idxs { + m[i] = &pilosa.IndexInfo{} + decodeIndex(idxs[i], m[i]) + } +} + +func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { + m.Name = idx.Name + m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) + decodeFields(idx.Fields, m.Fields) +} + +func decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { + for i := range fs { + m[i] = &pilosa.FieldInfo{} + decodeField(fs[i], m[i]) + } +} + +func decodeField(f *internal.Field, m *pilosa.FieldInfo) { + m.Name = f.Name + m.Options = pilosa.FieldOptions{} + decodeFieldOptions(f.Meta, &m.Options) + m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) + for _, viewname := range f.Views { + m.Views = append(m.Views, &pilosa.ViewInfo{Name: viewname}) + } +} + +func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) { + m.Type = options.Type + m.CacheType = options.CacheType + m.CacheSize = options.CacheSize + m.Min = options.Min + m.Max = options.Max + m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + m.Keys = options.Keys +} + +func decodeNodes(a []*internal.Node, m []*pilosa.Node) { + for i := range a { + m[i] = &pilosa.Node{} + decodeNode(a[i], m[i]) + } +} + +func decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) { + m.State = cs.State + m.ClusterID = cs.ClusterID + m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) + decodeNodes(cs.Nodes, m.Nodes) +} + +func decodeNode(node *internal.Node, m *pilosa.Node) { + m.ID = node.ID + decodeURI(node.URI, &m.URI) + m.IsCoordinator = node.IsCoordinator +} + +func decodeURI(i *internal.URI, m *pilosa.URI) { + m.Scheme = i.Scheme + m.Host = i.Host + m.Port = uint16(i.Port) +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage, m *pilosa.CreateShardMessage) { + m.Index = pb.Index + m.Shard = pb.Shard +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { + m.Index = pb.Index + m.Meta = &pilosa.IndexOptions{} + decodeIndexMeta(pb.Meta, m.Meta) +} + +func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { + m.Keys = pb.Keys +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { + m.Index = pb.Index +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.Meta = &pilosa.FieldOptions{} + decodeFieldOptions(pb.Meta, m.Meta) +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *pilosa.DeleteViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) { + m.JobID = pb.JobID + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.Error = pb.Error +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) { + m.NodeID = pb.NodeID + m.State = pb.State +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) { + m.Event = pilosa.NodeEventType(pb.Event) + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) +} + +func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.MaxShards = pb.MaxShards.Standard + m.Schema = &pilosa.Schema{} + decodeSchema(pb.Schema, m.Schema) +} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} + +func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { + m.Query = pb.Query + m.Shards = pb.Shards + m.ColumnAttrs = pb.ColumnAttrs + m.Remote = pb.Remote + m.ExcludeRowAttrs = pb.ExcludeRowAttrs + m.ExcludeColumns = pb.ExcludeColumns +} + +func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs + m.RowKeys = pb.RowKeys + m.ColumnKeys = pb.ColumnKeys + m.Timestamps = pb.Timestamps +} + +func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.ColumnIDs = pb.ColumnIDs + m.ColumnKeys = pb.ColumnKeys + m.Values = pb.Values +} + +func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { + m.Err = pb.Err +} + +func decodeBlockDataRequest(pb *internal.BlockDataRequest, m *pilosa.BlockDataRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View + m.Shard = pb.Shard + m.Block = pb.Block +} + +func decodeBlockDataResponse(pb *internal.BlockDataResponse, m *pilosa.BlockDataResponse) { + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs +} + +func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { + m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) + decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) + if pb.Err == "" { + m.Err = nil + } else { + m.Err = errors.New(pb.Err) + } + m.Results = make([]interface{}, len(pb.Results)) + decodeQueryResults(pb.Results, m.Results) + +} + +func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { + for i := range pb { + m[i] = &pilosa.ColumnAttrSet{} + decodeColumnAttrSet(pb[i], m[i]) + } +} + +func decodeColumnAttrSet(pb *internal.ColumnAttrSet, m *pilosa.ColumnAttrSet) { + m.ID = pb.ID + m.Key = pb.Key + m.Attrs = decodeAttrs(pb.Attrs) +} + +func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { + for i := range pb { + m[i] = decodeQueryResult(pb[i]) + } +} + +// QueryResult types. +const ( + queryResultTypeNil uint32 = iota + queryResultTypeRow + queryResultTypePairs + queryResultTypeValCount + queryResultTypeUint64 + queryResultTypeBool +) + +func decodeQueryResult(pb *internal.QueryResult) interface{} { + switch pb.Type { + case queryResultTypeRow: + return decodeRow(pb.Row) + case queryResultTypePairs: + return decodePairs(pb.Pairs) + case queryResultTypeValCount: + return decodeValCount(pb.ValCount) + case queryResultTypeUint64: + return pb.N + case queryResultTypeBool: + return pb.Changed + case queryResultTypeNil: + return nil + } + panic(fmt.Sprintf("unknown type: %d", pb.Type)) +} + +// DecodeRow converts r from its internal representation. +func decodeRow(pr *internal.Row) *pilosa.Row { + if pr == nil { + return nil + } + + r := pilosa.NewRow() + r.Attrs = decodeAttrs(pr.Attrs) + for _, v := range pr.Columns { + r.SetBit(v) + } + return r +} + +func decodeAttrs(pb []*internal.Attr) map[string]interface{} { + m := make(map[string]interface{}, len(pb)) + for i := range pb { + key, value := decodeAttr(pb[i]) + m[key] = value + } + return m +} + +const ( + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 +) + +func decodeAttr(attr *internal.Attr) (key string, value interface{}) { + switch attr.Type { + case attrTypeString: + return attr.Key, attr.StringValue + case attrTypeInt: + return attr.Key, attr.IntValue + case attrTypeBool: + return attr.Key, attr.BoolValue + case attrTypeFloat: + return attr.Key, attr.FloatValue + default: + return attr.Key, nil + } +} + +func decodePairs(a []*internal.Pair) []pilosa.Pair { + other := make([]pilosa.Pair, len(a)) + for i := range a { + other[i] = decodePair(a[i]) + } + return other +} + +func decodePair(pb *internal.Pair) pilosa.Pair { + return pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + } +} + +func decodeValCount(pb *internal.ValCount) pilosa.ValCount { + return pilosa.ValCount{ + Val: pb.Val, + Count: pb.Count, + } +} + +func EncodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) + for i := range a { + other[i] = EncodeColumnAttrSet(a[i]) + } + return other +} + +func EncodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), + } +} + +func EncodeRow(r *pilosa.Row) *internal.Row { + if r == nil { + return nil + } + + return &internal.Row{ + Columns: r.Columns(), + Attrs: encodeAttrs(r.Attrs), + } +} + +func EncodePairs(a pilosa.Pairs) []*internal.Pair { + other := make([]*internal.Pair, len(a)) + for i := range a { + other[i] = encodePair(a[i]) + } + return other +} + +func encodePair(p pilosa.Pair) *internal.Pair { + return &internal.Pair{ + ID: p.ID, + Key: p.Key, + Count: p.Count, + } +} + +func EncodeValCount(vc pilosa.ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +func encodeAttrs(m map[string]interface{}) []*internal.Attr { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + a := make([]*internal.Attr, len(keys)) + for i := range keys { + a[i] = encodeAttr(keys[i], m[keys[i]]) + } + return a +} + +// encodeAttr converts a key/value pair into an Attr internal representation. +func encodeAttr(key string, value interface{}) *internal.Attr { + pb := &internal.Attr{Key: key} + switch value := value.(type) { + case string: + pb.Type = attrTypeString + pb.StringValue = value + case float64: + pb.Type = attrTypeFloat + pb.FloatValue = value + case uint64: + pb.Type = attrTypeInt + pb.IntValue = int64(value) + case int64: + pb.Type = attrTypeInt + pb.IntValue = value + case bool: + pb.Type = attrTypeBool + pb.BoolValue = value + } + return pb +} diff --git a/event.go b/event.go index aa1e0e890..0d5e59e99 100644 --- a/event.go +++ b/event.go @@ -23,8 +23,8 @@ const ( NodeUpdate ) -// nodeEvent is a single event related to node activity in the cluster. -type nodeEvent struct { +// NodeEvent is a single event related to node activity in the cluster. +type NodeEvent struct { Event NodeEventType Node *Node } diff --git a/executor.go b/executor.go index 1a81e92bd..b39e26f58 100644 --- a/executor.go +++ b/executor.go @@ -20,7 +20,6 @@ import ( "sort" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -184,9 +183,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) case "Set": - return e.executeSetBit(ctx, index, c, opt) - case "SetValue": - return nil, e.executeSetValue(ctx, index, c, opt) + return e.executeSet(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -337,7 +334,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "map reduce") } // Attach attributes for Row() calls. @@ -1060,8 +1057,8 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetBit executes a Set() call. -func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { +// executeSet executes a Set() call. +func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Set() argument required: field") @@ -1077,14 +1074,7 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, ErrFieldNotFound } - // Read fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - + // Read colID using labels. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { return false, fmt.Errorf("reading Set() column: %v", err) @@ -1092,20 +1082,40 @@ func (e *executor) executeSetBit(ctx context.Context, index string, c *pql.Call, return false, fmt.Errorf("Set() column argument '%v' required", columnLabel) } - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) + if f.Type() == FieldTypeInt { + // Read remaining fields using labels. + rowVal, ok, err := c.IntArg(fieldName) if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) } - timestamp = &t - } - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) + } else { + // Read remaining fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + } } -// executeSetBitField executes a Set() call for a specific view. +// executeSetBitField executes a Set() call for a specific field. func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { shard := colID / ShardWidth ret := false @@ -1137,64 +1147,36 @@ func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql. return ret, nil } -// executeSetValue executes a SetValue() call. -func (e *executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { - // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) - if err != nil { - return fmt.Errorf("reading SetValue() column: %v", err) - } else if !ok { - return fmt.Errorf("SetValue() column field '%v' required", columnLabel) - } +// executeSetValueField executes a Set() call for a specific int field. +func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { + shard := colID / ShardWidth + ret := false - // Copy args and remove reserved fields. - args := pql.CopyArgs(c.Args) - // While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) - - // Set values. - for name, value := range args { - // Retrieve field. - field := e.Holder.Field(index, name) - if field == nil { - return ErrFieldNotFound - } - - switch value := value.(type) { - case int64: - if _, err := field.SetValue(columnID, value); err != nil { - return err + for _, node := range e.Cluster.shardNodes(index, shard) { + // Update locally if host matches. + if node.ID == e.Node.ID { + val, err := f.SetValue(colID, value) + if err != nil { + return false, err + } else if val { + ret = true } - default: - return ErrInvalidBSIGroupValueType + continue } - field.Stats.Count("SetValue", 1, 1.0) - } - // Do not forward call if this is already being forwarded. - if opt.Remote { - return nil - } + // Do not forward call if this is already being forwarded. + if opt.Remote { + continue + } - // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) - resp := make(chan error, len(nodes)) - for _, node := range nodes { - go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) - resp <- err - }(node) - } - - // Return first error. - for range nodes { - if err := <-resp; err != nil { - return err + // Forward call to remote node otherwise. + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + return false, err + } else { + ret = res[0].(bool) } } - - return nil + return ret, nil } // executeSetRowAttrs executes a SetRowAttrs() call. @@ -1392,7 +1374,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // exec executes a PQL query remotely for a set of shards on a node. func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) { // Encode request object. - pbreq := &internal.QueryRequest{ + pbreq := &QueryRequest{ Query: q.String(), Shards: shards, Remote: true, @@ -1403,40 +1385,7 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q * return nil, err } - // Return an error, if specified on response. - if err := decodeError(pb.Err); err != nil { - return nil, err - } - - // Return appropriate data for the query. - results = make([]interface{}, len(q.Calls)) - for i, call := range q.Calls { - var v interface{} - var err error - - switch call.Name { - case "Average", "Sum": - v, err = decodeValCount(pb.Results[i].GetValCount()), nil - case "TopN": - v, err = decodePairs(pb.Results[i].GetPairs()), nil - case "Count": - v, err = pb.Results[i].N, nil - case "Set": - v, err = pb.Results[i].Changed, nil - case "Clear": - v, err = pb.Results[i].Changed, nil - case "SetRowAttrs": - case "SetColumnAttrs": - default: - v, err = DecodeRow(pb.Results[i].GetRow()), nil - } - if err != nil { - return nil, err - } - - results[i] = v - } - return results, nil + return pb.Results, pb.Err } // shardsByNode returns a mapping of nodes to shards. @@ -1490,7 +1439,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, for { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, errors.Wrap(ctx.Err(), "context done") case resp := <-ch: // On error retry against remaining nodes. If an error returns then // the context will cancel and cause all open goroutines to return. @@ -1500,10 +1449,10 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { - return nil, err + return nil, errors.Wrap(err, "calling mapper") } continue } @@ -1524,7 +1473,7 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return err + return errors.Wrap(err, "shards by node") } // Execute each node in a separate goroutine. @@ -1771,20 +1720,6 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func EncodeValCount(vc ValCount) *internal.ValCount { - return &internal.ValCount{ - Val: vc.Val, - Count: vc.Count, - } -} - -func decodeValCount(pb *internal.ValCount) ValCount { - return ValCount{ - Val: pb.Val, - Count: pb.Count, - } -} - // Smaller returns the smaller of the two ValCounts. func (vc *ValCount) Smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { diff --git a/executor_test.go b/executor_test.go index f54fb7e33..41341b814 100644 --- a/executor_test.go +++ b/executor_test.go @@ -39,7 +39,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateField("f", pilosa.FieldOptions{}) + f, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } @@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) - if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) } @@ -354,7 +354,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if err := index.DeleteField("f"); err != nil { t.Fatal(err) } - if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } @@ -365,7 +365,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Run("ErrInvalidRowValueType", func(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{}) - if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) } if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `row value must be a string when field 'keys' option enabled` { @@ -385,7 +385,7 @@ func TestExecutor_Execute_OldPQL(t *testing.T) { hldr.SetBit("i", "f", 1, 0) if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { - t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err)) + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err) } } @@ -398,20 +398,16 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 50, - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { t.Fatal(err) - } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } // Set bsiGroup values. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { t.Fatal(err) } @@ -439,28 +435,24 @@ func TestExecutor_Execute_SetValue(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 100, - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { t.Fatal(err) } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) @@ -475,9 +467,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } @@ -514,9 +506,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) @@ -555,9 +547,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("a", f="foo") @@ -741,15 +733,11 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: -10, - Max: 100, - }); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { t.Fatal(err) } @@ -760,14 +748,14 @@ func TestExecutor_Execute_MinMax(t *testing.T) { Set(1, x=1) Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) - SetValue(col=0, f=20) - SetValue(col=1, f=-5) - SetValue(col=2, f=-5) - SetValue(col=3, f=10) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60) + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) `}); err != nil { t.Fatal(err) } @@ -836,31 +824,19 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 10, - Max: 100, - }); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 100000, - }); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 1000, - }); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { t.Fatal(err) } @@ -868,13 +844,13 @@ func TestExecutor_Execute_Sum(t *testing.T) { Set(0, x=0) Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - SetValue(col=0, foo=20) - SetValue(col=0, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) `}); err != nil { t.Fatal(err) } @@ -906,10 +882,7 @@ func TestExecutor_Execute_Range(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeTime, - TimeQuantum: pilosa.TimeQuantum("YMDH"), - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { t.Fatal(err) } @@ -962,39 +935,23 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Fatal(err) } - if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("foo", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 10, - Max: 100, - }); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("bar", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 100000, - }); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("other", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 1000, - }); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { t.Fatal(err) } - if _, err := idx.CreateField("edge", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 100, - }); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { t.Fatal(err) } @@ -1002,15 +959,15 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) - SetValue(col=50, foo=20) - SetValue(col=50, bar=2000) - SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) - SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10) - SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) - SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) - SetValue(col=0, other=1000) - SetValue(col=0, edge=100) - SetValue(col=1, edge=-100) + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) `}); err != nil { t.Fatal(err) } @@ -1268,7 +1225,7 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := index.CreateField("f", pilosa.FieldOptions{}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatalf("creating field: %v", err) } @@ -1352,10 +1309,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { indexName := strings.ToLower(string(tt.quantum)) index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeTime, - TimeQuantum: tt.quantum, - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil { t.Fatal(err) } // Populate diff --git a/field.go b/field.go index 2e4897f91..bcdb800f8 100644 --- a/field.go +++ b/field.go @@ -75,9 +75,28 @@ type Field struct { Logger Logger } -// FieldOption is a functional option type for pilosa.FieldOptions. +// FieldOption is a functional option type for pilosa.fieldOptions. type FieldOption func(fo *FieldOptions) error +func OptFieldKeys() FieldOption { + return func(fo *FieldOptions) error { + fo.Keys = true + return nil + } +} + +func OptFieldTypeDefault() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeSet + fo.CacheType = DefaultCacheType + fo.CacheSize = DefaultCacheSize + return nil + } +} + func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { @@ -120,12 +139,19 @@ func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption { } // NewField returns a new instance of field. -func NewField(path, index, name string, options FieldOptions) (*Field, error) { +func NewField(path, index, name string, opts FieldOption) (*Field, error) { err := validateName(name) if err != nil { return nil, err } + // Apply functional option. + fo := FieldOptions{} + err = opts(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + f := &Field{ path: path, index: index, @@ -138,7 +164,7 @@ func NewField(path, index, name string, options FieldOptions) (*Field, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - options: applyDefaultOptions(options), + options: applyDefaultOptions(fo), Logger: NopLogger, } @@ -178,13 +204,6 @@ func (f *Field) Type() string { return f.options.Type } -// CacheType returns the caching mode for the field. -func (f *Field) CacheType() string { - f.mu.RLock() - defer f.mu.RUnlock() - return f.options.CacheType -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Field) SetCacheSize(v uint32) error { @@ -605,7 +624,7 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) { if created { // Broadcast view creation to the cluster. err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ + &CreateViewMessage{ Index: f.index, Field: f.name, View: name, @@ -1073,25 +1092,6 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { return nil } -// encodeFields converts a into its internal representation. -func encodeFields(a []*Field) []*internal.Field { - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -// encodeField converts f into its internal representation. -func encodeField(f *Field) *internal.Field { - fo := f.options - return &internal.Field{ - Name: f.name, - Meta: fo.Encode(), - Views: f.viewNames(), - } -} - type fieldSlice []*Field func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1102,7 +1102,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } type FieldInfo struct { Name string `json:"name"` Options FieldOptions `json:"options"` - Views []*viewInfo `json:"views,omitempty"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo @@ -1119,7 +1119,7 @@ type FieldOptions struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Keys bool `json:"keys,omitempty"` + Keys bool `json:"keys"` } // applyDefaultOptions returns a new FieldOptions object @@ -1155,21 +1155,6 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { } } -func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { - if options == nil { - return nil - } - return &FieldOptions{ - Type: options.Type, - CacheType: options.CacheType, - CacheSize: options.CacheSize, - Min: options.Min, - Max: options.Max, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Keys: options.Keys, - } -} - func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { case FieldTypeSet: @@ -1177,28 +1162,34 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { Type string `json:"type"` CacheType string `json:"cacheType"` CacheSize uint32 `json:"cacheSize"` + Keys bool `json:"keys"` }{ o.Type, o.CacheType, o.CacheSize, + o.Keys, }) case FieldTypeInt: return json.Marshal(struct { Type string `json:"type"` Min int64 `json:"min"` Max int64 `json:"max"` + Keys bool `json:"keys"` }{ o.Type, o.Min, o.Max, + o.Keys, }) case FieldTypeTime: return json.Marshal(struct { Type string `json:"type"` TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` }{ o.Type, o.TimeQuantum, + o.Keys, }) } return nil, errors.New("invalid field type") diff --git a/field_internal_test.go b/field_internal_test.go index a11b5c394..4ded2bebe 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -153,7 +153,7 @@ func TestBSIGroup_BaseValue(t *testing.T) { // Ensure field can open and retrieve a view. func TestField_DeleteView(t *testing.T) { - f := MustOpenField(FieldOptions{}) + f := MustOpenField(OptFieldTypeDefault()) defer f.Close() viewName := viewStandard + "_v" @@ -190,12 +190,12 @@ type TestField struct { } // NewTestField returns a new instance of TestField d/0. -func NewTestField(options FieldOptions) *TestField { +func NewTestField(opts FieldOption) *TestField { path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - field, err := NewField(path, "i", "f", options) + field, err := NewField(path, "i", "f", opts) if err != nil { panic(err) } @@ -203,8 +203,8 @@ func NewTestField(options FieldOptions) *TestField { } // MustOpenField returns a new, opened field at a temporary path. Panic on error. -func MustOpenField(options FieldOptions) *TestField { - f := NewTestField(options) +func MustOpenField(opts FieldOption) *TestField { + f := NewTestField(opts) if err := f.Open(); err != nil { panic(err) } @@ -225,7 +225,7 @@ func (f *TestField) Reopen() error { } path, index, name := f.Path(), f.Index(), f.Name() - f.Field, err = NewField(path, index, name, FieldOptions{}) + f.Field, err = NewField(path, index, name, OptFieldTypeDefault()) if err != nil { return err } @@ -253,7 +253,7 @@ func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) { // Ensure field can open and retrieve a view. func TestField_CreateViewIfNotExists(t *testing.T) { - f := MustOpenField(FieldOptions{}) + f := MustOpenField(OptFieldTypeDefault()) defer f.Close() // Create view. @@ -278,7 +278,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) { } func TestField_SetTimeQuantum(t *testing.T) { - f := MustOpenField(FieldOptions{Type: FieldTypeTime}) + f := MustOpenField(OptFieldTypeTime(TimeQuantum(""))) defer f.Close() // Set & retrieve time quantum. @@ -297,7 +297,7 @@ func TestField_SetTimeQuantum(t *testing.T) { } func TestField_RowTime(t *testing.T) { - f := MustOpenField(FieldOptions{Type: FieldTypeTime}) + f := MustOpenField(OptFieldTypeTime(TimeQuantum(""))) defer f.Close() if err := f.SetTimeQuantum(TimeQuantum("YMDH")); err != nil { diff --git a/field_test.go b/field_test.go index 4b1def96c..520179f52 100644 --- a/field_test.go +++ b/field_test.go @@ -28,11 +28,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 30, - }) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) if err != nil { t.Fatal(err) } @@ -65,11 +61,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 30, - }) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) if err != nil { t.Fatal(err) } @@ -102,9 +94,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeSet, - }) + f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } @@ -119,11 +109,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 20, - Max: 30, - }) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) if err != nil { t.Fatal(err) } @@ -138,11 +124,7 @@ func TestField_SetValue(t *testing.T) { idx := test.MustOpenIndex() defer idx.Close() - f, err := idx.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 20, - Max: 30, - }) + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) if err != nil { t.Fatal(err) } @@ -159,7 +141,7 @@ func TestField_NameRestriction(t *testing.T) { if err != nil { panic(err) } - field, err := pilosa.NewField(path, "i", ".meta", pilosa.FieldOptions{}) + field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault()) if field != nil { t.Fatalf("unexpected field name %s", err) } @@ -191,13 +173,13 @@ func TestField_NameValidation(t *testing.T) { panic(err) } for _, name := range validFieldNames { - _, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{}) + _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault()) if err != nil { t.Fatalf("unexpected field name: %s %s", name, err) } } for _, name := range invalidFieldNames { - _, err := pilosa.NewField(path, "i", name, pilosa.FieldOptions{}) + _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault()) if err == nil { t.Fatalf("expected error on field name: %s", name) } diff --git a/fragment.go b/fragment.go index b502e1597..1ae3c5bb3 100644 --- a/fragment.go +++ b/fragment.go @@ -1889,7 +1889,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { } // Execute query. - queryRequest := &internal.QueryRequest{ + queryRequest := &QueryRequest{ Query: buffers[k].String(), Remote: true, } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index eda4cbbdc..4ceb33819 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -750,7 +750,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { defer index.Close() // Create field. - field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked, CacheSize: cacheSize}) + field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, cacheSize)) if err != nil { t.Fatal(err) } @@ -916,7 +916,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { defer index.Close() // Create field. - field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked}) + field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) if err != nil { t.Fatal(err) } diff --git a/gossip/gossip.go b/gossip/gossip.go index 4f562af5e..7a11d01d4 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -15,6 +15,8 @@ package gossip import ( + "bytes" + "context" "fmt" "io/ioutil" "log" @@ -24,10 +26,8 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -42,8 +42,8 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - pserver pilosa.MemberServer - config *gossipConfig + papi *pilosa.API + config *gossipConfig Logger pilosa.Logger @@ -145,9 +145,10 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - host := s.Node().URI.Host() +func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...GossipMemberSetOption) (*GossipMemberSet, error) { + host := api.Node().URI.Host g := &GossipMemberSet{ + papi: api, Logger: pilosa.NopLogger, } @@ -157,7 +158,7 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet return nil, errors.Wrap(err, "executing option") } } - ger := newGossipEventReceiver(g.logger, s) + ger := newGossipEventReceiver(g.logger, api) g.gossipEventReceiver = ger if g.transport == nil { @@ -189,11 +190,11 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet // memberlist config conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.Net - conf.Name = s.Node().ID - conf.BindAddr = s.Node().URI.Host() + conf.Name = api.Node().ID + conf.BindAddr = api.Node().URI.Host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(s.Node().URI.Host()) + conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -214,14 +215,12 @@ func NewGossipMemberSet(cfg Config, s *pilosa.Server, options ...GossipMemberSet gossipSeeds: cfg.Seeds, } - g.pserver = s - return g, nil } // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := proto.Marshal(pilosa.EncodeNode(g.pserver.Node())) + buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -232,14 +231,9 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte { // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. func (g *GossipMemberSet) NotifyMsg(b []byte) { - m, err := pilosa.UnmarshalMessage(b) + err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b)) if err != nil { - g.Logger.Printf("unmarshal message error: %s", err) - return - } - if err := g.pserver.ReceiveMessage(m); err != nil { - g.Logger.Printf("receive message error: %s", err) - return + g.Logger.Printf("cluster message error: %s", err) } } @@ -252,14 +246,14 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. func (g *GossipMemberSet) LocalState(join bool) []byte { - pb, err := g.pserver.LocalStatus() - if err != nil { - g.Logger.Printf("error getting local state, err=%s", err) - return []byte{} + m := &pilosa.NodeStatus{ + Node: g.papi.Node(), + MaxShards: g.papi.MaxShards(context.Background()), + Schema: &pilosa.Schema{Indexes: g.papi.Holder().Schema()}, } // Marshal nodestate data to bytes. - buf, err := proto.Marshal(pb) + buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -270,13 +264,7 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { // MergeRemoteState implementation of the memberlist.Delegate interface // receive and process the remote side's LocalState. func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { - // Unmarshal nodestate data. - var pb internal.NodeStatus - if err := proto.Unmarshal(buf, &pb); err != nil { - g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) - return - } - err := g.pserver.HandleRemoteStatus(&pb) + err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)) if err != nil { g.Logger.Printf("merge state error: %s", err) } @@ -288,18 +276,18 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type gossipEventReceiver struct { - ch chan memberlist.NodeEvent - eventHandler *pilosa.Server + ch chan memberlist.NodeEvent + papi *pilosa.API logger *log.Logger } // newGossipEventReceiver returns a new instance of GossipEventReceiver. -func newGossipEventReceiver(logger *log.Logger, pserver *pilosa.Server) *gossipEventReceiver { +func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { ger := &gossipEventReceiver{ - ch: make(chan memberlist.NodeEvent, 1), - logger: logger, - eventHandler: pserver, + ch: make(chan memberlist.NodeEvent, 1), + logger: logger, + papi: papi, } go ger.listen() return ger @@ -333,16 +321,20 @@ func (g *gossipEventReceiver) listen() { } // Get the node from the event.Node meta data. - var n internal.Node - if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { - panic("failed to unmarshal event node meta data") + var n pilosa.Node + if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { + panic("failed to unmarshal event node meta into node") } - ne := &internal.NodeEventMessage{ - Event: uint32(nodeEventType), + ne := &pilosa.NodeEvent{ + Event: nodeEventType, Node: &n, } - if err := g.eventHandler.ReceiveMessage(ne); err != nil { + buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer) + if err != nil { + panic(err) + } + if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil { g.logger.Printf("receive event error: %s", err) } } diff --git a/handler.go b/handler.go index 1f3d04300..9fc3af368 100644 --- a/handler.go +++ b/handler.go @@ -75,3 +75,40 @@ func (n nopHandler) Close() error { } var NopHandler Handler = nopHandler{} + +type ImportValueRequest struct { + Index string + Field string + Shard uint64 + ColumnIDs []uint64 + ColumnKeys []string + Values []int64 +} + +type ImportRequest struct { + Index string + Field string + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +type ImportResponse struct { + Err string +} + +type BlockDataRequest struct { + Index string + Field string + View string + Shard uint64 + Block uint64 +} + +type BlockDataResponse struct { + RowIDs []uint64 + ColumnIDs []uint64 +} diff --git a/holder.go b/holder.go index bc255abba..192c83a46 100644 --- a/holder.go +++ b/holder.go @@ -27,7 +27,6 @@ import ( "syscall" "time" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -217,7 +216,7 @@ func (h *Holder) Schema() []*IndexInfo { for _, field := range index.Fields() { fi := &FieldInfo{Name: field.Name(), Options: field.Options()} for _, view := range field.views() { - fi.Views = append(fi.Views, &viewInfo{Name: view.name}) + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) di.Fields = append(di.Fields, fi) @@ -229,8 +228,24 @@ func (h *Holder) Schema() []*IndexInfo { return a } +// limitedSchema returns schema information for all indexes and fields. +func (h *Holder) limitedSchema() []*IndexInfo { + var a []*IndexInfo + for _, index := range h.Indexes() { + di := &IndexInfo{Name: index.Name()} + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + di.Fields = append(di.Fields, fi) + } + sort.Sort(fieldInfoSlice(di.Fields)) + a = append(a, di) + } + sort.Sort(indexInfoSlice(a)) + return a +} + // applySchema applies an internal Schema to Holder. -func (h *Holder) applySchema(schema *internal.Schema) error { +func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. for _, index := range schema.Indexes { opt := IndexOptions{} @@ -240,14 +255,13 @@ func (h *Holder) applySchema(schema *internal.Schema) error { } // Create fields that don't exist. for _, f := range index.Fields { - opt := decodeFieldOptions(f.Meta) - field, err := idx.CreateFieldIfNotExists(f.Name, *opt) + field, err := idx.createFieldIfNotExists(f.Name, f.Options) if err != nil { return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v) + _, err := field.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -257,20 +271,6 @@ func (h *Holder) applySchema(schema *internal.Schema) error { return nil } -// encodeMaxShards creates and internal representation of max shards. -func (h *Holder) encodeMaxShards() *internal.MaxShards { - return &internal.MaxShards{ - Standard: h.maxShards(), - } -} - -// encodeSchema creates an internal representation of schema. -func (h *Holder) encodeSchema() *internal.Schema { - return &internal.Schema{ - Indexes: encodeIndexes(h.Indexes()), - } -} - // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/holder_internal_test.go b/holder_internal_test.go index 425873005..75a3508fd 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -60,7 +60,7 @@ func newHolder() *tHolder { // MustCreateFieldIfNotExists returns a given field. Panic on error. func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field { - f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, FieldOptions{}) + f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault()) if err != nil { panic(err) } @@ -105,7 +105,7 @@ func TestHolder_Optn(t *testing.T) { if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { t.Fatal(err) - } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { t.Fatal(err) @@ -129,7 +129,7 @@ func TestHolder_Optn(t *testing.T) { if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { t.Fatal(err) - } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { t.Fatal(err) @@ -154,7 +154,7 @@ func TestHolder_Optn(t *testing.T) { if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { t.Fatal(err) - } else if field, err := idx.CreateField("bar", FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if view, err := field.createViewIfNotExists(viewStandard); err != nil { t.Fatal(err) diff --git a/holder_test.go b/holder_test.go index 078ab55f3..d4886ff30 100644 --- a/holder_test.go +++ b/holder_test.go @@ -98,7 +98,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -117,7 +117,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -135,7 +135,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -157,7 +157,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := field.SetBit(0, 0, nil); err != nil { t.Fatal(err) @@ -178,7 +178,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := field.SetBit(0, 0, nil); err != nil { t.Fatal(err) diff --git a/http/client.go b/http/client.go index 27597a5cb..7453c87fd 100644 --- a/http/client.go +++ b/http/client.go @@ -27,22 +27,15 @@ import ( "sort" "strconv" - "crypto/tls" - - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - // InternalClient represents a client to the Pilosa cluster. type InternalClient struct { defaultURI *pilosa.URI + serializer pilosa.Serializer // The client to use for HTTP communication. HTTPClient *http.Client @@ -66,13 +59,11 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { return &InternalClient{ defaultURI: defaultURI, + serializer: proto.Serializer{}, HTTPClient: remoteClient, } } -// Host returns the host the client was initialized with. -func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } - // MaxShardByIndex returns the number of shards on a server by index. func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { return c.maxShardByIndex(ctx) @@ -217,22 +208,21 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard } // Query executes query against the index. -func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { return c.QueryNode(ctx, c.defaultURI, index, queryRequest) } // QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { if index == "" { return nil, pilosa.ErrIndexRequired } else if queryRequest.Query == "" { return nil, pilosa.ErrQueryRequired } - // Encode request object. - buf, err := proto.Marshal(queryRequest) + buf, err := c.serializer.Marshal(queryRequest) if err != nil { - return nil, errors.Wrap(err, "marshaling") + return nil, errors.Wrap(err, "marshaling queryRequest") } // Create HTTP request. @@ -262,11 +252,11 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s return nil, errors.New(string(body)) } - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { + qresp := &pilosa.QueryResponse{} + if err := c.serializer.Unmarshal(body, qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) + } else if qresp.Err != nil { + return nil, qresp.Err } return qresp, nil @@ -280,7 +270,7 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard return pilosa.ErrFieldRequired } - buf, err := marshalImportPayload(index, field, shard, bits) + buf, err := c.marshalImportPayload(index, field, shard, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -309,7 +299,7 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, colum return pilosa.ErrFieldRequired } - buf, err := marshalImportPayloadK(index, field, columns) + buf, err := c.marshalImportPayloadK(index, field, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -343,14 +333,14 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, Shard: shard, @@ -365,14 +355,14 @@ func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { +func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowKeys := Bits(bits).RowKeys() columnKeys := Bits(bits).ColumnKeys() timestamps := Bits(bits).Timestamps() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ Index: index, Field: field, RowKeys: rowKeys, @@ -414,8 +404,8 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, inde return errors.New(string(body)) } - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { + var isresp pilosa.ImportResponse + if err := c.serializer.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) } else if s := isresp.Err; s != "" { return errors.New(s) @@ -432,7 +422,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s return pilosa.ErrFieldRequired } - buf, err := marshalImportValuePayload(index, field, shard, vals) + buf, err := c.marshalImportValuePayload(index, field, shard, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -454,13 +444,13 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { +func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ + buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{ Index: index, Field: field, Shard: shard, @@ -683,7 +673,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, if uri == nil { panic("need to pass a URI to BlockData") } - buf, err := proto.Marshal(&internal.BlockDataRequest{ + buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ Index: index, Field: field, Shard: shard, @@ -719,10 +709,10 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, } // Decode response object. - var rsp internal.BlockDataResponse + var rsp pilosa.BlockDataResponse if body, err := ioutil.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { + } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { return nil, nil, errors.Wrap(err, "unmarshalling") } return rsp.RowIDs, rsp.ColumnIDs, nil @@ -819,12 +809,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index } // SendMessage posts a message synchronously. -func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { u := uriPathToURL(uri, "/internal/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) if err != nil { @@ -998,7 +983,7 @@ func pos(rowID, columnID uint64) uint64 { func uriPathToURL(uri *pilosa.URI, path string) url.URL { return url.URL{ - Scheme: uri.Scheme(), + Scheme: uri.Scheme, Host: uri.HostPort(), Path: path, } @@ -1006,7 +991,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL { func nodePathToURL(node *pilosa.Node, path string) url.URL { return url.URL{ - Scheme: node.URI.Scheme(), + Scheme: node.URI.Scheme, Host: node.URI.HostPort(), Path: path, } diff --git a/http/client_test.go b/http/client_test.go index fb1105a27..2944a7586 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -24,7 +24,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -131,7 +130,7 @@ func TestClient_MultiNode(t *testing.T) { client[2] = MustNewClient(c[2].URL(), defaultClient) topN := 4 - queryRequest := &internal.QueryRequest{ + queryRequest := &pilosa.QueryRequest{ Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } @@ -147,17 +146,17 @@ func TestClient_MultiNode(t *testing.T) { } // Test must return exactly N results. - if len(result.Results[0].Pairs) != topN { + if len(result.Results[0].([]pilosa.Pair)) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } - p := []*internal.Pair{ + p := []pilosa.Pair{ {ID: 100, Count: 12}, {ID: 22, Count: 10}, {ID: 98, Count: 8}, {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].Pairs, p) { + if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } @@ -218,15 +217,10 @@ func TestClient_ImportValue(t *testing.T) { hldr := test.Holder{Holder: holder} fldName := "f" - fo := pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 100, - } // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - field, err := index.CreateFieldIfNotExists(fldName, fo) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) if err != nil { t.Fatal(err) } diff --git a/http/handler.go b/http/handler.go index 9d6b9e080..a221b1445 100644 --- a/http/handler.go +++ b/http/handler.go @@ -33,11 +33,9 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -358,9 +356,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } schema := h.API.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: schema, - }); err != nil { + if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { h.Logger.Printf("write schema response error: %s", err) } } @@ -374,7 +370,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { status := getStatusResponse{ State: h.API.State(), Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.LocalID(), + LocalID: h.API.Node().ID, } if err := json.NewEncoder(w).Encode(status); err != nil { h.Logger.Printf("write status response error: %s", err) @@ -416,18 +412,25 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.API.Query(r.Context(), req) if err != nil { - w.WriteHeader(http.StatusBadRequest) + switch errors.Cause(resp.Err) { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusBadRequest) + } h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) return } - // Set appropriate status code, if there is an error. + // Set appropriate status code, if there is an error. It doesn't appear that + // resp.Err could ever be set in API.Query, so this code block is probably + // doing nothing right now. if resp.Err != nil { - switch resp.Err { + switch errors.Cause(resp.Err) { case pilosa.ErrTooManyWrites: w.WriteHeader(http.StatusRequestEntityTooLarge) default: - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusBadRequest) } } @@ -655,17 +658,22 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { } // Convert json options into functional options. - var fos pilosa.FieldOption + var fos []pilosa.FieldOption switch req.Options.Type { case pilosa.FieldTypeSet: - fos = pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize) + fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) case pilosa.FieldTypeInt: - fos = pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max) + fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) case pilosa.FieldTypeTime: - fos = pilosa.OptFieldTypeTime(*req.Options.TimeQuantum) + fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) + } + if req.Options.Keys != nil { + if *req.Options.Keys { + fos = append(fos, pilosa.OptFieldKeys()) + } } - _, err = h.API.CreateField(r.Context(), indexName, fieldName, fos) + _, err = h.API.CreateField(r.Context(), indexName, fieldName, fos...) resp.write(w, err) } @@ -817,13 +825,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryReques return nil, errors.Wrap(err, "reading") } - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") + qreq := &pilosa.QueryRequest{} + err = h.API.Serializer.Unmarshal(body, qreq) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling query request") } - - return decodeQueryRequest(&req), nil + return qreq, nil } // readURLQueryRequest parses query parameters from URL parameters from r. @@ -862,7 +869,7 @@ func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res // writeProtobufQueryResponse writes the response from the executor to w as protobuf. func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + if buf, err := h.API.Serializer.Marshal(resp); err != nil { return errors.Wrap(err, "marshalling") } else if _, err := w.Write(buf); err != nil { return errors.Wrap(err, "writing") @@ -914,8 +921,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if field.Type() == pilosa.FieldTypeInt { // Field type: Int // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportValueRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -932,8 +939,8 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } else { // Field type: Set, Time // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { + req := &pilosa.ImportRequest{} + if err := h.API.Serializer.Unmarshal(body, req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -950,7 +957,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: ""}) + buf, e := h.API.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) return @@ -1105,56 +1112,6 @@ const ( QueryResultTypeBool ) -func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { - req := &pilosa.QueryRequest{ - Query: pb.Query, - Shards: pb.Shards, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - -func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *pilosa.Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = pilosa.EncodeRow(result) - case []pilosa.Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = pilosa.EncodePairs(result) - case pilosa.ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = pilosa.EncodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb -} - // parseUint64Slice returns a slice of uint64s from a comma-delimited string. func parseUint64Slice(s string) ([]uint64, error) { var a []uint64 @@ -1326,10 +1283,6 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } } -func (h *Handler) GetAPI() *pilosa.API { - return h.API -} - type defaultClusterMessageResponse struct{} func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { diff --git a/http/translator_test.go b/http/translator_test.go index a3a5e8603..04531dab5 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -75,7 +75,7 @@ func TestTranslateStore_Reader(t *testing.T) { defer main.Close() // Connect to server and stream all available data. - store := http.NewTranslateStore(main.Server.URI.String()) + store := http.NewTranslateStore(main.URL()) rc, err := store.Reader(context.Background(), 100) if err != nil { @@ -128,7 +128,7 @@ func TestTranslateStore_Reader(t *testing.T) { // Connect to server and begin streaming. ctx, cancel := context.WithCancel(context.Background()) - store := http.NewTranslateStore(main.Server.URI.String()) + store := http.NewTranslateStore(main.URL()) if _, err := store.Reader(ctx, 0); err != nil { t.Fatal(err) } @@ -155,7 +155,7 @@ func TestTranslateStore_Reader(t *testing.T) { main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] defer main.Close() - _, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0) + _, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0) if err != pilosa.ErrNotImplemented { t.Fatalf("unexpected error: %s", err) } diff --git a/index.go b/index.go index d1a6a8ee5..7217bb659 100644 --- a/index.go +++ b/index.go @@ -270,7 +270,7 @@ func (i *Index) RecalculateCaches() { } // CreateField creates a field. -func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) { +func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() @@ -278,11 +278,40 @@ func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) { if i.fields[name] != nil { return nil, NewConflictError(ErrFieldExists) } - return i.createField(name, opt) + + // Apply functional options. + fo := FieldOptions{} + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return i.createField(name, fo) } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. -func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) { +func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Find field in cache first. + if f := i.fields[name]; f != nil { + return f, nil + } + + // Apply functional option. + fo := FieldOptions{} + err := opts(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + + return i.createField(name, fo) +} + +func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() @@ -330,7 +359,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { } func (i *Index) newField(path, name string) (*Field, error) { - f, err := NewField(path, i.name, name, FieldOptions{}) // TODO: NewField should be un-exported along with FieldOptions + f, err := NewField(path, i.name, name, OptFieldTypeDefault()) if err != nil { return nil, err } @@ -376,8 +405,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { - Name string `json:"name"` - Fields []*FieldInfo `json:"fields"` + Name string `json:"name"` + Options IndexOptions `json:"options"` + Fields []*FieldInfo `json:"fields"` } type indexInfoSlice []*IndexInfo @@ -386,35 +416,11 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// encodeIndexes converts a into its internal representation. -func encodeIndexes(a []*Index) []*internal.Index { - other := make([]*internal.Index, len(a)) - for i := range a { - other[i] = encodeIndex(a[i]) - } - return other -} - -// encodeIndex converts d into its internal representation. -func encodeIndex(d *Index) *internal.Index { - return &internal.Index{ - Name: d.name, - Fields: encodeFields(d.Fields()), - } -} - // IndexOptions represents options to set when initializing an index. type IndexOptions struct { Keys bool `json:"keys"` } -// Encode converts i into its internal representation. -func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - Keys: i.Keys, - } -} - // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/index_test.go b/index_test.go index 5490e2978..bc412646a 100644 --- a/index_test.go +++ b/index_test.go @@ -33,7 +33,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) { defer index.Close() // Create field. - f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) + f, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } else if f == nil { @@ -41,7 +41,7 @@ func TestIndex_CreateFieldIfNotExists(t *testing.T) { } // Retrieve existing field. - other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) + other, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } else if f.Field != other.Field { @@ -61,10 +61,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with explicit quantum. - f, err := index.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeTime, - TimeQuantum: pilosa.TimeQuantum("YMDH"), - }) + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { @@ -80,11 +77,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with schema and verify it exists. - if f, err := index.CreateField("f", pilosa.FieldOptions{ - Type: pilosa.FieldTypeInt, - Min: 10, - Max: 20, - }); err != nil { + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { t.Fatalf("unexpected type: %#v", f.Type()) @@ -184,7 +177,7 @@ func TestIndex_DeleteField(t *testing.T) { defer index.Close() // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } diff --git a/iterator_test.go b/iterator_internal_test.go similarity index 100% rename from iterator_test.go rename to iterator_internal_test.go diff --git a/pilosa.go b/pilosa.go index ebc2be438..9615e88b8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,8 +17,6 @@ package pilosa import ( "errors" "regexp" - - "github.com/pilosa/pilosa/internal" ) // System errors. @@ -122,23 +120,6 @@ type ColumnAttrSet struct { Attrs map[string]interface{} `json:"attrs,omitempty"` } -// EncodeColumnAttrSets converts a into its internal representation. -func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { - other := make([]*internal.ColumnAttrSet, len(a)) - for i := range a { - other[i] = EncodeColumnAttrSet(a[i]) - } - return other -} - -// EncodeColumnAttrSet converts set into its internal representation. -func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { - return &internal.ColumnAttrSet{ - ID: set.ID, - Attrs: encodeAttrs(set.Attrs), - } -} - // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" diff --git a/pql/ast.go b/pql/ast.go index 0bcc582d4..47baa9d6c 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -220,23 +220,6 @@ func (q *Query) WriteCallN() int { return n } -// HasKeys returns true if any call in the query uses keys and requires translation to ids. -func (q *Query) HasKeys() bool { - for _, call := range q.Calls { - if call.Args["col"] != nil { - if _, ok := call.Args["col"].(string); ok { - return true - } - } - if call.Args["row"] != nil { - if _, ok := call.Args["row"].(string); ok { - return true - } - } - } - return false -} - // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -285,6 +268,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } +// IntArg is for reading the value at key from call.Args as an int64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a unt64 or an int64 and +// then cast to an int64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) IntArg(key string) (int64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return tval, true, nil + case uint64: + return int64(tval), true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval) + } +} + // UintSliceArg reads the value at key from call.Args as a slice of uint64. If // the key is not in Call.Args, the value of the returned bool will be false, // and the error will be nil. If the value is a slice of int64 it will convert @@ -309,22 +312,6 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } -// StringArg is for reading the value at key from call.Args as a string. If the -// key is not in Call.Args, the value of the returned bool will be false, and -// the error will be nil. An error is returned if the value is not a string. -func (c *Call) StringArg(key string) (string, bool, error) { - val, ok := c.Args[key] - if !ok { - return "", false, nil - } - switch tval := val.(type) { - case string: - return tval, true, nil - default: - return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) - } -} - // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/pql/pql.peg b/pql/pql.peg index a33031543..141d8f265 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -5,7 +5,7 @@ type PQL Peg { } -Calls <- whitesp (Call whitesp)* !. +Calls <- sp (Call sp)* !. Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()} / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} @@ -62,11 +62,10 @@ col <- ( {p.addPosNum("_col", buffer[begin:end])} open <- '(' sp close <- ')' sp -sp <- ( ' ' / '\t' )* -comma <- sp ',' whitesp +sp <- ( ' ' / '\t' / '\n' )* +comma <- sp ',' sp lbrack <- '[' sp rbrack <- sp ']' sp -whitesp <- ( ' ' / '\t' / '\n' )* IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 697d589c3..2516def6c 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -45,7 +45,6 @@ const ( rulecomma rulelbrack rulerbrack - rulewhitesp ruleIDENT ruletimestampbasicfmt ruletimestampfmt @@ -128,7 +127,6 @@ var rul3s = [...]string{ "comma", "lbrack", "rbrack", - "whitesp", "IDENT", "timestampbasicfmt", "timestampfmt", @@ -294,7 +292,7 @@ type PQL struct { Buffer string buffer []rune - rules [80]func() bool + rules [79]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -548,12 +546,12 @@ func (p *PQL) Init() { _rules = [...]func() bool{ nil, - /* 0 Calls <- <(whitesp (Call whitesp)* !.)> */ + /* 0 Calls <- <(sp (Call sp)* !.)> */ func() bool { position0, tokenIndex0 := position, tokenIndex { position1 := position - if !_rules[rulewhitesp]() { + if !_rules[rulesp]() { goto l0 } l2: @@ -562,7 +560,7 @@ func (p *PQL) Init() { if !_rules[ruleCall]() { goto l3 } - if !_rules[rulewhitesp]() { + if !_rules[rulesp]() { goto l3 } goto l2 @@ -2525,7 +2523,7 @@ func (p *PQL) Init() { position, tokenIndex = position257, tokenIndex257 return false }, - /* 25 sp <- <(' ' / '\t')*> */ + /* 25 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { position260 := position @@ -2542,6 +2540,13 @@ func (p *PQL) Init() { l264: position, tokenIndex = position263, tokenIndex263 if buffer[position] != rune('\t') { + goto l265 + } + position++ + goto l263 + l265: + position, tokenIndex = position263, tokenIndex263 + if buffer[position] != rune('\n') { goto l262 } position++ @@ -2555,295 +2560,258 @@ func (p *PQL) Init() { } return true }, - /* 26 comma <- <(sp ',' whitesp)> */ + /* 26 comma <- <(sp ',' sp)> */ func() bool { - position265, tokenIndex265 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position266 := position + position267 := position if !_rules[rulesp]() { - goto l265 + goto l266 } if buffer[position] != rune(',') { - goto l265 + goto l266 } position++ - if !_rules[rulewhitesp]() { - goto l265 + if !_rules[rulesp]() { + goto l266 } - add(rulecomma, position266) + add(rulecomma, position267) } return true - l265: - position, tokenIndex = position265, tokenIndex265 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 27 lbrack <- <('[' sp)> */ nil, /* 28 rbrack <- <(sp ']' sp)> */ nil, - /* 29 whitesp <- <(' ' / '\t' / '\n')*> */ - func() bool { - { - position270 := position - l271: - { - position272, tokenIndex272 := position, tokenIndex - { - position273, tokenIndex273 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l274 - } - position++ - goto l273 - l274: - position, tokenIndex = position273, tokenIndex273 - if buffer[position] != rune('\t') { - goto l275 - } - position++ - goto l273 - l275: - position, tokenIndex = position273, tokenIndex273 - if buffer[position] != rune('\n') { - goto l272 - } - position++ - } - l273: - goto l271 - l272: - position, tokenIndex = position272, tokenIndex272 - } - add(rulewhitesp, position270) - } - return true - }, - /* 30 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, - /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position271, tokenIndex271 := position, tokenIndex { - position278 := position + position272 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if buffer[position] != rune('-') { - goto l277 + goto l271 } position++ { - position279, tokenIndex279 := position, tokenIndex + position273, tokenIndex273 := position, tokenIndex if buffer[position] != rune('0') { - goto l280 + goto l274 } position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 + goto l273 + l274: + position, tokenIndex = position273, tokenIndex273 if buffer[position] != rune('1') { - goto l277 + goto l271 } position++ } - l279: + l273: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if buffer[position] != rune('-') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if buffer[position] != rune('T') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if buffer[position] != rune(':') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l271 } position++ - add(ruletimestampbasicfmt, position278) + add(ruletimestampbasicfmt, position272) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l271: + position, tokenIndex = position271, tokenIndex271 return false }, - /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position281, tokenIndex281 := position, tokenIndex + position275, tokenIndex275 := position, tokenIndex { - position282 := position + position276 := position { - position283, tokenIndex283 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex if buffer[position] != rune('"') { - goto l284 + goto l278 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l284 + goto l278 } if buffer[position] != rune('"') { - goto l284 + goto l278 } position++ - goto l283 - l284: - position, tokenIndex = position283, tokenIndex283 + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 if buffer[position] != rune('\'') { - goto l285 + goto l279 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l285 + goto l279 } if buffer[position] != rune('\'') { - goto l285 + goto l279 } position++ - goto l283 - l285: - position, tokenIndex = position283, tokenIndex283 + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 if !_rules[ruletimestampbasicfmt]() { - goto l281 + goto l275 } } - l283: - add(ruletimestampfmt, position282) + l277: + add(ruletimestampfmt, position276) } return true - l281: - position, tokenIndex = position281, tokenIndex281 + l275: + position, tokenIndex = position275, tokenIndex275 return false }, - /* 33 timestamp <- <( Action43)> */ + /* 32 timestamp <- <( Action43)> */ nil, - /* 35 Action0 <- <{p.startCall("Set")}> */ + /* 34 Action0 <- <{p.startCall("Set")}> */ nil, - /* 36 Action1 <- <{p.endCall()}> */ + /* 35 Action1 <- <{p.endCall()}> */ nil, - /* 37 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 38 Action3 <- <{p.endCall()}> */ + /* 37 Action3 <- <{p.endCall()}> */ nil, - /* 39 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, - /* 40 Action5 <- <{p.endCall()}> */ + /* 39 Action5 <- <{p.endCall()}> */ nil, - /* 41 Action6 <- <{p.startCall("Clear")}> */ + /* 40 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 42 Action7 <- <{p.endCall()}> */ + /* 41 Action7 <- <{p.endCall()}> */ nil, - /* 43 Action8 <- <{p.startCall("TopN")}> */ + /* 42 Action8 <- <{p.startCall("TopN")}> */ nil, - /* 44 Action9 <- <{p.endCall()}> */ + /* 43 Action9 <- <{p.endCall()}> */ nil, - /* 45 Action10 <- <{p.startCall("Range")}> */ + /* 44 Action10 <- <{p.startCall("Range")}> */ nil, - /* 46 Action11 <- <{p.endCall()}> */ + /* 45 Action11 <- <{p.endCall()}> */ nil, nil, - /* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 49 Action13 <- <{ p.endCall() }> */ + /* 48 Action13 <- <{ p.endCall() }> */ nil, - /* 50 Action14 <- <{ p.addBTWN() }> */ + /* 49 Action14 <- <{ p.addBTWN() }> */ nil, - /* 51 Action15 <- <{ p.addLTE() }> */ + /* 50 Action15 <- <{ p.addLTE() }> */ nil, - /* 52 Action16 <- <{ p.addGTE() }> */ + /* 51 Action16 <- <{ p.addGTE() }> */ nil, - /* 53 Action17 <- <{ p.addEQ() }> */ + /* 52 Action17 <- <{ p.addEQ() }> */ nil, - /* 54 Action18 <- <{ p.addNEQ() }> */ + /* 53 Action18 <- <{ p.addNEQ() }> */ nil, - /* 55 Action19 <- <{ p.addLT() }> */ + /* 54 Action19 <- <{ p.addLT() }> */ nil, - /* 56 Action20 <- <{ p.addGT() }> */ + /* 55 Action20 <- <{ p.addGT() }> */ nil, - /* 57 Action21 <- <{p.startConditional()}> */ + /* 56 Action21 <- <{p.startConditional()}> */ nil, - /* 58 Action22 <- <{p.endConditional()}> */ + /* 57 Action22 <- <{p.endConditional()}> */ nil, - /* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 64 Action28 <- <{ p.startList() }> */ + /* 63 Action28 <- <{ p.startList() }> */ nil, - /* 65 Action29 <- <{ p.endList() }> */ + /* 64 Action29 <- <{ p.endList() }> */ nil, - /* 66 Action30 <- <{ p.addVal(nil) }> */ + /* 65 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 67 Action31 <- <{ p.addVal(true) }> */ + /* 66 Action31 <- <{ p.addVal(true) }> */ nil, - /* 68 Action32 <- <{ p.addVal(false) }> */ + /* 67 Action32 <- <{ p.addVal(false) }> */ nil, - /* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + /* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 79 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 78 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index f472e2cc2..ad40364b3 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -228,6 +228,13 @@ func TestPEGWorking(t *testing.T) { name: "Dashed Frame", input: "Set(1, my-frame=9)", ncalls: 1}, + { + name: "newlines", + input: `Set( +1, +my-frame +=9)`, + ncalls: 1}, } for i, test := range tests { diff --git a/row.go b/row.go index cbfa6b270..d6a0037cc 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) @@ -216,25 +215,6 @@ func (r *Row) InvalidateCount() { } } -// IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased. -func (r *Row) IncrementCount(i uint64) { - seg := r.segment(i / ShardWidth) - if seg != nil { - seg.n++ - } - -} - -// DecrementCount decrements the row cached counter. -func (r *Row) DecrementCount(i uint64) { - seg := r.segment(i / ShardWidth) - if seg != nil { - if seg.n > 0 { - seg.n-- - } - } -} - // Count returns the number of columns in the row. func (r *Row) Count() uint64 { var n uint64 @@ -271,41 +251,6 @@ func (r *Row) Columns() []uint64 { return a } -// EncodeRow converts r into its internal representation. -func EncodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// DecodeRow converts r from its internal representation. -func DecodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - -// Union performs a union on a slice of rows. -func Union(rows []*Row) *Row { - other := rows[0] - for _, r := range rows[1:] { - other = other.Union(r) - } - return other -} - // RowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The // width of the segment will always match the shard width. diff --git a/server.go b/server.go index 7fbd94066..3adf3bb50 100644 --- a/server.go +++ b/server.go @@ -27,8 +27,6 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -36,12 +34,11 @@ import ( // Default server settings. const ( - DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" + defaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Ensure Server implements interfaces. var _ broadcaster = &Server{} -var _ MemberServer = &Server{} // Server represents a holder wrapped by a running HTTP server. type Server struct { @@ -57,6 +54,7 @@ type Server struct { executor *executor hosts []string clusterDisabled bool + serializer Serializer // External systemInfo SystemInfo @@ -64,7 +62,7 @@ type Server struct { logger Logger nodeID string - URI URI + uri URI antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration @@ -188,7 +186,7 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { func OptServerURI(uri *URI) ServerOption { return func(s *Server) error { - s.URI = *uri + s.uri = *uri return nil } } @@ -203,6 +201,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +func OptServerSerializer(ser Serializer) ServerOption { + return func(s *Server) error { + s.serializer = ser + return nil + } +} + func OptServerIsCoordinator(is bool) ServerOption { return func(s *Server) error { s.isCoordinator = is @@ -230,7 +235,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { closing: make(chan struct{}), cluster: newCluster(), holder: NewHolder(), - diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), + diagnostics: NewDiagnosticsCollector(defaultDiagnosticServer), systemInfo: NewNopSystemInfo(), gcNotifier: NopGCNotifier, @@ -277,7 +282,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Set Cluster Node. node := &Node{ ID: s.nodeID, - URI: s.URI, + URI: s.uri, IsCoordinator: s.cluster.Coordinator == s.nodeID, } s.cluster.Node = node @@ -431,41 +436,41 @@ func (s *Server) monitorAntiEntropy() { } } -// ReceiveMessage represents an implementation of BroadcastHandler. -func (s *Server) ReceiveMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateShardMessage: +// receiveMessage represents an implementation of BroadcastHandler. +func (s *Server) receiveMessage(m Message) error { + switch obj := m.(type) { + case *CreateShardMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } idx.setRemoteMaxShard(obj.Shard) - case *internal.CreateIndexMessage: + case *CreateIndexMessage: opt := IndexOptions{} _, err := s.holder.CreateIndex(obj.Index, opt) if err != nil { return err } - case *internal.DeleteIndexMessage: + case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFieldMessage: + case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFieldOptions(obj.Meta) - _, err := idx.CreateField(obj.Field, *opt) + opt := obj.Meta + _, err := idx.createField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFieldMessage: + case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { return err } - case *internal.CreateViewMessage: + case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -474,7 +479,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.DeleteViewMessage: + case *DeleteViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) @@ -483,52 +488,59 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } - case *internal.ClusterStatus: + case *ClusterStatus: err := s.cluster.mergeClusterStatus(obj) if err != nil { return err } - case *internal.ResizeInstruction: + case *ResizeInstruction: err := s.cluster.followResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: err := s.cluster.markResizeInstructionComplete(obj) if err != nil { return err } - case *internal.SetCoordinatorMessage: - s.cluster.setCoordinator(DecodeNode(obj.New)) - case *internal.UpdateCoordinatorMessage: - s.cluster.updateCoordinator(DecodeNode(obj.New)) - case *internal.NodeStateMessage: + case *SetCoordinatorMessage: + s.cluster.setCoordinator(obj.New) + case *UpdateCoordinatorMessage: + s.cluster.updateCoordinator(obj.New) + case *NodeStateMessage: err := s.cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } - case *internal.RecalculateCaches: + case *RecalculateCaches: s.holder.RecalculateCaches() - case *internal.NodeEventMessage: - s.cluster.ReceiveEvent(DecodeNodeEvent(obj)) + case *NodeEvent: + s.cluster.ReceiveEvent(obj) + case *NodeStatus: + s.handleRemoteStatus(obj) } return nil } // SendSync represents an implementation of Broadcaster. -func (s *Server) SendSync(pb proto.Message) error { +func (s *Server) SendSync(m Message) error { var eg errgroup.Group + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) for _, node := range s.cluster.Nodes { node := node s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. - if s.URI == node.URI { + if s.uri == node.URI { continue } eg.Go(func() error { - return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, msg) }) } @@ -536,72 +548,48 @@ func (s *Server) SendSync(pb proto.Message) error { } // SendAsync represents an implementation of Broadcaster. -func (s *Server) SendAsync(pb proto.Message) error { +func (s *Server) SendAsync(m Message) error { return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. -func (s *Server) SendTo(to *Node, pb proto.Message) error { +func (s *Server) SendTo(to *Node, m Message) error { s.logger.Printf("SendTo: %s", to.URI) - return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) + return s.defaultClient.SendMessage(context.Background(), &to.URI, msg) } -// Node returns the pilosa.Node object. It is used by membership protocols to +// node returns the pilosa.node object. It is used by membership protocols to // get this node's name(ID), location(URI), and coordinator status. -func (s *Server) Node() *Node { - return s.cluster.Node +func (s *Server) node() Node { + return *s.cluster.Node } -// Server implements StatusHandler. -// LocalStatus is used to periodically sync information -// between nodes. Under normal conditions, nodes should -// remain in sync through Broadcast messages. For cases -// where a node fails to receive a Broadcast message, or -// when a new (empty) node needs to get in sync with the -// rest of the cluster, two things are shared via gossip: -// - MaxShard by Index -// - Schema -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) LocalStatus() (proto.Message, error) { - if s.cluster == nil { - return nil, errors.New("Server.Cluster is nil") - } - if s.holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Node: EncodeNode(s.cluster.Node), - MaxShards: s.holder.encodeMaxShards(), - Schema: s.holder.encodeSchema(), - } - - return &ns, nil -} - -// HandleRemoteStatus receives incoming NodeStatus from remote nodes. -func (s *Server) HandleRemoteStatus(pb proto.Message) error { +// handleRemoteStatus receives incoming NodeStatus from remote nodes. +func (s *Server) handleRemoteStatus(pb Message) { // Ignore NodeStatus messages until the cluster is in a Normal state. if s.cluster.State() != ClusterStateNormal { - return nil + return } go func() { // Make sure the holder has opened. <-s.holder.opened - err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + err := s.mergeRemoteStatus(pb.(*NodeStatus)) if err != nil { s.logger.Printf("merge remote status: %s", err) } }() - - return nil } -func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { +func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { // Ignore status updates from self. - if s.nodeID == DecodeNode(ns.Node).ID { + if s.nodeID == ns.Node.ID { return nil } @@ -612,7 +600,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // Sync maxShards. oldmaxshards := s.holder.maxShards() - for index, newMax := range ns.MaxShards.Standard { + for index, newMax := range ns.MaxShards { localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because // indexes should be created and synced prior to shard creation @@ -641,7 +629,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.URI.host) + s.diagnostics.Set("Host", s.uri.Host) s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) @@ -756,10 +744,3 @@ func expandDirName(path string) (string, error) { } return path, nil } - -type MemberServer interface { - ReceiveMessage(proto.Message) error - LocalStatus() (proto.Message, error) - HandleRemoteStatus(proto.Message) error - Node() *Node -} diff --git a/server/config.go b/server/config.go index 55da45768..73f0b289e 100644 --- a/server/config.go +++ b/server/config.go @@ -21,13 +21,6 @@ import ( "github.com/pilosa/pilosa/toml" ) -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) diff --git a/server/handler_test.go b/server/handler_test.go index e49b1eb0f..9670df2c2 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -27,10 +27,8 @@ import ( gohttp "net/http" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -62,17 +60,17 @@ func TestHandler_Endpoints(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } @@ -144,7 +142,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Shards args protobuf", func(t *testing.T) { // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Count(Row(f0=30))", Shards: []uint64{0, 1}, }) @@ -196,13 +194,11 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 3 { - t.Fatalf("unexpected n: %d", n) + } else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) } }) @@ -244,27 +240,25 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + } else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } }) t.Run("Row columnattrs protobuf", func(t *testing.T) { // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ + buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ Query: "Row(f0=30)", ColumnAttrs: true, }) @@ -281,22 +275,22 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { + } else if _, ok := resp.Results[0].(*pilosa.Row); !ok { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) } if a := resp.ColumnAttrSets; len(a) != 2 { @@ -305,8 +299,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) + } else if a[0].Attrs["x"] != "y" { + t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"]) } }) @@ -329,12 +323,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { + } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } }) @@ -358,10 +350,10 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.Err; s != `executing: field not found` { + } else if s := resp.Err.Error(); s != `executing: field not found` { t.Fatalf("unexpected error: %s", s) } }) @@ -401,7 +393,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Field delete", func(t *testing.T) { i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { + if _, err := i.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } w := httptest.NewRecorder() @@ -453,7 +445,7 @@ func TestHandler_Endpoints(t *testing.T) { } }) - meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) + meta, err := i.CreateFieldIfNotExists("meta", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } diff --git a/server/server.go b/server/server.go index 164dc94e3..401da09e8 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" @@ -139,7 +140,7 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "opening server") } - m.logger.Printf("Listening as %s\n", m.Server.URI) + m.logger.Printf("Listening as %s\n", m.API.Node().URI) return nil } @@ -202,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.Scheme() == "https" { + if uri.Scheme == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -235,7 +236,7 @@ func (m *Command) SetupServer() error { } // If port is 0, get auto-allocated port from listener - if uri.Port() == 0 { + if uri.Port == 0 { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } @@ -271,6 +272,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerSerializer(proto.Serializer{}), coordinatorOpt, } @@ -309,7 +311,7 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.Server.URI.Host() + gossipHost := m.API.Node().URI.Host m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -317,7 +319,7 @@ func (m *Command) SetupNetworking() error { gossipMemberSet, err := gossip.NewGossipMemberSet( m.Config.Gossip, - m.Server, + m.API, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(m.gossipTransport), ) @@ -366,19 +368,19 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.Scheme() == "https" && tlsconf != nil { + if uri.Scheme == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.Scheme() == "http" { + } else if uri.Scheme == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme) } return ln, nil diff --git a/server/server_test.go b/server/server_test.go index 8b1e29acc..506807f70 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -44,7 +44,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } diff --git a/test/field.go b/test/field.go index 345deadf8..7a0439026 100644 --- a/test/field.go +++ b/test/field.go @@ -28,12 +28,12 @@ type Field struct { } // NewField returns a new instance of Field d/0. -func NewField(options pilosa.FieldOptions) *Field { +func NewField(opts pilosa.FieldOption) *Field { path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - field, err := pilosa.NewField(path, "i", "f", options) + field, err := pilosa.NewField(path, "i", "f", opts) if err != nil { panic(err) } @@ -41,8 +41,8 @@ func NewField(options pilosa.FieldOptions) *Field { } // MustOpenField returns a new, opened field at a temporary path. Panic on error. -func MustOpenField(options pilosa.FieldOptions) *Field { - f := NewField(options) +func MustOpenField(opts pilosa.FieldOption) *Field { + f := NewField(opts) if err := f.Open(); err != nil { panic(err) } @@ -63,7 +63,7 @@ func (f *Field) Reopen() error { } path, index, name := f.Path(), f.Index(), f.Name() - f.Field, err = pilosa.NewField(path, index, name, pilosa.FieldOptions{}) + f.Field, err = pilosa.NewField(path, index, name, pilosa.OptFieldTypeDefault()) if err != nil { return err } @@ -76,7 +76,7 @@ func (f *Field) Reopen() error { // Ensure field can set its cache func TestField_SetCacheSize(t *testing.T) { - f := MustOpenField(pilosa.FieldOptions{}) + f := MustOpenField(pilosa.OptFieldTypeDefault()) defer f.Close() cacheSize := uint32(100) diff --git a/test/holder.go b/test/holder.go index 94d1c778f..ff87ae02c 100644 --- a/test/holder.go +++ b/test/holder.go @@ -81,19 +81,10 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption return &Index{Index: idx} } -// MustCreateFieldIfNotExists returns a given field. Panic on error. -func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{}) - if err != nil { - panic(err) - } - return f -} - // Row returns a Row for a given field. func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } @@ -106,7 +97,7 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } @@ -115,7 +106,7 @@ func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum string) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } @@ -129,7 +120,7 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum // SetBit clears a bit on the given field. func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } @@ -142,7 +133,7 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { // ClearBit clears a bit on the given field. func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } diff --git a/test/index.go b/test/index.go index 69c18682c..9b7c1684b 100644 --- a/test/index.go +++ b/test/index.go @@ -74,8 +74,8 @@ func (i *Index) Reopen() error { } // CreateField creates a field with the given options. -func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) { - f, err := i.Index.CreateField(name, opt) +func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, error) { + f, err := i.Index.CreateField(name, opts...) if err != nil { return nil, err } @@ -83,8 +83,8 @@ func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. -func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) { - f, err := i.Index.CreateFieldIfNotExists(name, opt) +func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) { + f, err := i.Index.CreateFieldIfNotExists(name, opts) if err != nil { return nil, err } diff --git a/test/pilosa.go b/test/pilosa.go index 4774b4cf8..d56723e89 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -63,11 +63,6 @@ func NewCommand(opts ...server.CommandOption) *Command { m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr - err = m.SetupServer() - if err != nil { - panic(err) - } - if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) @@ -117,10 +112,6 @@ func (m *Command) Reopen() error { config := m.Command.Config m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...) m.Command.Config = config - err := m.SetupServer() - if err != nil { - return errors.Wrap(err, "setting up server") - } // Run new program. if err := m.Start(); err != nil { @@ -130,11 +121,11 @@ func (m *Command) Reopen() error { } // URL returns the base URL string for accessing the running program. -func (m *Command) URL() string { return m.Server.URI.String() } +func (m *Command) URL() string { return m.API.Node().URI.String() } // Client returns a client to connect to the program. func (m *Command) Client() *http.InternalClient { - client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } diff --git a/time.go b/time.go index def889304..ecfdcad3d 100644 --- a/time.go +++ b/time.go @@ -70,15 +70,6 @@ func (q TimeQuantum) Type() string { return "TimeQuantum" } -// ParseTimeQuantum parses v into a time quantum. -func ParseTimeQuantum(v string) (TimeQuantum, error) { - q := TimeQuantum(strings.ToUpper(v)) - if !q.Valid() { - return "", ErrInvalidTimeQuantum - } - return q, nil -} - // viewByTimeUnit returns the view name for time with a given quantum unit. func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { diff --git a/time_internal_test.go b/time_internal_test.go index 2920685fc..bd4afae43 100644 --- a/time_internal_test.go +++ b/time_internal_test.go @@ -16,6 +16,7 @@ package pilosa import ( "reflect" + "strings" "testing" "time" ) @@ -23,7 +24,7 @@ import ( // Ensure string can be parsed into time quantum. func TestParseTimeQuantum(t *testing.T) { t.Run("OK", func(t *testing.T) { - if q, err := ParseTimeQuantum("YMDH"); err != nil { + if q, err := parseTimeQuantum("YMDH"); err != nil { t.Fatalf("unexpected error: %s", err) } else if q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %#v", q) @@ -31,7 +32,7 @@ func TestParseTimeQuantum(t *testing.T) { }) t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { - if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { + if _, err := parseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { t.Fatalf("unexpected error: %s", err) } }) @@ -160,9 +161,18 @@ func mustParseTime(value string) time.Time { // mustParseTimeQuantum parses v into a time quantum. Panic on error. func mustParseTimeQuantum(v string) TimeQuantum { - q, err := ParseTimeQuantum(v) + q, err := parseTimeQuantum(v) if err != nil { panic(err) } return q } + +// parseTimeQuantum parses v into a time quantum. +func parseTimeQuantum(v string) (TimeQuantum, error) { + q := TimeQuantum(strings.ToUpper(v)) + if !q.Valid() { + return "", ErrInvalidTimeQuantum + } + return q, nil +} diff --git a/uri.go b/uri.go index 5d823d8b6..8577c8238 100644 --- a/uri.go +++ b/uri.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -43,17 +42,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a- // localhost // :10101 type URI struct { - scheme string `json:"scheme"` - host string `json:"host"` - port uint16 `json:"port"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` } // DefaultURI creates and returns the default URI. func DefaultURI() *URI { return &URI{ - scheme: "http", - host: "localhost", - port: 10101, + Scheme: "http", + Host: "localhost", + Port: 10101, } } @@ -83,44 +82,29 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// Scheme returns the scheme of this URI. -func (u *URI) Scheme() string { - return u.scheme -} - // SetScheme sets the scheme of this URI. func (u *URI) SetScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) if m == nil { return errors.New("invalid scheme") } - u.scheme = scheme + u.Scheme = scheme return nil } -// Host returns the host of this URI. -func (u *URI) Host() string { - return u.host -} - // SetHost sets the host of this URI. func (u *URI) SetHost(host string) error { m := hostRegexp.FindStringSubmatch(host) if m == nil { return errors.New("invalid host") } - u.host = host + u.Host = host return nil } -// Port returns the port of this URI. -func (u *URI) Port() uint16 { - return u.port -} - // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { - u.port = port + u.Port = port } // HostPort returns `Host:Port` @@ -129,31 +113,23 @@ func (u *URI) HostPort() string { if u == nil { return "" } - s := fmt.Sprintf("%s:%d", u.host, u.port) + s := fmt.Sprintf("%s:%d", u.Host, u.Port) return s } // Normalize returns the address in a form usable by a HTTP client. func (u *URI) Normalize() string { - scheme := u.scheme + scheme := u.Scheme index := strings.Index(scheme, "+") if index >= 0 { scheme = scheme[:index] } - return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port) } // String returns the address as a string. func (u URI) String() string { - return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) -} - -// Equals returns true if the checked URI is equivalent to this URI. -func (u URI) Equals(other *URI) bool { - if other == nil { - return false - } - return u == *other + return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port) } // Path returns URI with path @@ -163,7 +139,7 @@ func (u *URI) Path(path string) string { // The following methods are required to implement pflag Value interface. -// Set sets the time quantum value. +// Set sets the uri value. func (u *URI) Set(value string) error { uri, err := NewURIFromAddress(value) if err != nil { @@ -173,7 +149,7 @@ func (u *URI) Set(value string) error { return nil } -// Type returns the type of a time quantum value. +// Type returns the type of a uri. func (u URI) Type() string { return "URI" } @@ -199,41 +175,13 @@ func parseAddress(address string) (uri *URI, err error) { } } uri = &URI{ - scheme: scheme, - host: host, - port: uint16(port), + Scheme: scheme, + Host: host, + Port: uint16(port), } return uri, nil } -// Encode converts o into its internal representation. -func (u URI) Encode() *internal.URI { - return encodeURI(u) -} - -func encodeURI(u URI) *internal.URI { - return &internal.URI{ - Scheme: u.scheme, - Host: u.host, - Port: uint32(u.port), - } -} - -func DecodeURI(i *internal.URI) URI { - return decodeURI(i) -} - -func decodeURI(i *internal.URI) URI { - if i == nil { - return URI{} - } - return URI{ - scheme: i.Scheme, - host: i.Host, - port: uint16(i.Port), - } -} - // MarshalJSON marshals URI into a JSON-encoded byte slice. func (u *URI) MarshalJSON() ([]byte, error) { var output struct { @@ -241,9 +189,9 @@ func (u *URI) MarshalJSON() ([]byte, error) { Host string `json:"host,omitempty"` Port uint16 `json:"port,omitempty"` } - output.Scheme = u.scheme - output.Host = u.host - output.Port = u.port + output.Scheme = u.Scheme + output.Host = u.Host + output.Port = u.Port return json.Marshal(output) } @@ -257,8 +205,8 @@ func (u *URI) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &input); err != nil { return err } - u.scheme = input.Scheme - u.host = input.Host - u.port = input.Port + u.Scheme = input.Scheme + u.Host = input.Host + u.Port = input.Port return nil } diff --git a/uri_test.go b/uri_internal_test.go similarity index 62% rename from uri_test.go rename to uri_internal_test.go index 70ea3f273..3c9631661 100644 --- a/uri_test.go +++ b/uri_internal_test.go @@ -1,34 +1,16 @@ // Copyright 2017 Pilosa Corp. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. +// http://www.apache.org/licenses/LICENSE-2.0 // -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND -// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -// DAMAGE. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package pilosa @@ -94,16 +76,6 @@ func TestURIPath(t *testing.T) { } } -func TestEquals(t *testing.T) { - uri1 := DefaultURI() - if uri1.Equals(nil) { - t.Fatalf("URI should not be equal to nil") - } - if !uri1.Equals(DefaultURI()) { - t.Fatalf("URI should be equal to another URI with the same scheme, host and port") - } -} - func TestSetScheme(t *testing.T) { uri := DefaultURI() target := "fun" @@ -111,8 +83,8 @@ func TestSetScheme(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Scheme() != target { - t.Fatalf("%s != %s", uri.Scheme(), target) + if uri.Scheme != target { + t.Fatalf("%s != %s", uri.Scheme, target) } } @@ -123,8 +95,8 @@ func TestSetHost(t *testing.T) { if err != nil { t.Fatal(err) } - if uri.Host() != target { - t.Fatalf("%s != %s", uri.host, target) + if uri.Host != target { + t.Fatalf("%s != %s", uri.Host, target) } } @@ -132,8 +104,8 @@ func TestSetPort(t *testing.T) { uri := DefaultURI() target := uint16(9999) uri.SetPort(target) - if uri.Port() != target { - t.Fatalf("%d != %d", uri.port, target) + if uri.Port != target { + t.Fatalf("%d != %d", uri.Port, target) } } @@ -165,14 +137,14 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.Scheme() != scheme { - t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme) + if uri.Scheme != scheme { + t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.Host() != host { - t.Fatalf("Host does not match: %s != %s", uri.host, host) + if uri.Host != host { + t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.Port() != port { - t.Fatalf("Port does not match: %d != %d", uri.port, port) + if uri.Port != port { + t.Fatalf("Port does not match: %d != %d", uri.Port, port) } } diff --git a/utils_internal_test.go b/utils_internal_test.go index ca7bd1fa7..df1b7d2e0 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,7 +24,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -104,13 +103,13 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } -func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error { +func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error { for _, c := range t.Clusters { idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { return err } - if _, err := idx.CreateField(field, opt); err != nil { + if _, err := idx.CreateField(field, opts); err != nil { return err } } @@ -162,7 +161,7 @@ func (t *ClusterCluster) addNode() error { // Send NodeJoin event to coordinator. if id > 0 { coord := t.Clusters[0] - ev := &nodeEvent{ + ev := &NodeEvent{ Event: NodeJoin, Node: c.Node, } @@ -304,9 +303,9 @@ func (t *ClusterCluster) Close() error { } // SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *ClusterCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: +func (t *ClusterCluster) SendSync(m Message) error { + switch obj := m.(type) { + case *ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { c.mergeClusterStatus(obj) @@ -322,19 +321,19 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { } // SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *ClusterCluster) SendAsync(pb proto.Message) error { +func (t *ClusterCluster) SendAsync(Message) error { return nil } // SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: +func (t *ClusterCluster) SendTo(to *Node, m Message) error { + switch obj := m.(type) { + case *ResizeInstruction: err := t.FollowResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: coord := t.clusterByID(to.ID) go coord.markResizeInstructionComplete(obj) } @@ -342,10 +341,10 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } // FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error { // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -356,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi // figure out which node it was meant for, then call the operation on that cluster // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) - instrNode := DecodeNode(instr.Node) + instrNode := instr.Node destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. @@ -365,8 +364,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi } for _, src := range instr.Sources { - srcNode := DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) + srcCluster := t.clusterByID(src.Node.ID) srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) @@ -405,6 +403,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi complete.Error = err.Error() } - node := DecodeNode(instr.Coordinator) + node := instr.Coordinator return t.SendTo(node, complete) } diff --git a/view.go b/view.go index fd5306b85..609664304 100644 --- a/view.go +++ b/view.go @@ -22,7 +22,6 @@ import ( "strings" "sync" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -232,7 +231,7 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { // Send the create shard message to all nodes. err := v.broadcaster.SendSync( - &internal.CreateShardMessage{ + &CreateShardMessage{ Index: v.index, Shard: shard, }) @@ -422,12 +421,12 @@ func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (* return r, nil } -// viewInfo represents schema information for a view. -type viewInfo struct { +// ViewInfo represents schema information for a view. +type ViewInfo struct { Name string `json:"name"` } -type viewInfoSlice []*viewInfo +type viewInfoSlice []*ViewInfo func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p viewInfoSlice) Len() int { return len(p) }