From 5dd7a9556a09e1d6c80172e996e75bf3660fa25e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 28 Jun 2018 14:06:37 -0500 Subject: [PATCH] rename slice to shard --- Gopkg.lock | 2 +- NOTES | 12 +- api.go | 82 ++++---- apimethod_string.go | 6 +- broadcast.go | 10 +- broadcast_test.go | 4 +- client.go | 32 +-- cluster.go | 64 +++--- cluster_internal_test.go | 24 +-- cmd/import.go | 2 +- ctl/export.go | 14 +- ctl/import.go | 22 +- diagnostics.go | 6 +- executor.go | 298 ++++++++++++++-------------- executor_test.go | 204 +++++++++---------- field.go | 20 +- fragment.go | 68 +++---- fragment_internal_test.go | 12 +- handler.go | 6 +- holder.go | 42 ++-- holder_test.go | 44 ++-- http/client.go | 110 +++++----- http/client_test.go | 56 +++--- http/handler.go | 66 +++--- index.go | 26 +-- index_test.go | 4 +- internal/private.pb.go | 408 ++++++++++++++++++++------------------ internal/private.proto | 12 +- internal/public.pb.go | 187 +++++++++-------- internal/public.proto | 6 +- iterator.go | 4 +- pilosa.go | 2 +- roaring/roaring_test.go | 24 +-- row.go | 44 ++-- row_test.go | 20 +- server.go | 22 +- server/cluster_test.go | 24 +-- server/handler_test.go | 50 ++--- stats_test.go | 20 +- test/fragment.go | 4 +- test/handler.go | 6 +- test/holder.go | 4 +- utils_internal_test.go | 14 +- view.go | 100 +++++----- view_internal_test.go | 12 +- 45 files changed, 1120 insertions(+), 1079 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d3a12ef8a..33187bfe2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -310,6 +310,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" + inputs-digest = "da6d02118ca77527c4ff00e9522880032fc052fb39bc8efe6c76602857c8c84e" solver-name = "gps-cdcl" solver-version = 1 diff --git a/NOTES b/NOTES index 6b8e088ea..8da55fe11 100644 --- a/NOTES +++ b/NOTES @@ -14,13 +14,13 @@ │0000000000000000000000000000000000000000│ │────────────────────────────────────────┤ F ▶│0000000000000000000000000000000000000000│ - r ││0000000000000000000000000000000000000000│ - a ││0000000000000000000000000000000000000000│ - m ││0000000000000000000000000000000000000000│ - e ▶│0000000000000000000000000000000000000000│ + i ││0000000000000000000000000000000000000000│ + e ││0000000000000000000000000000000000000000│ + l ││0000000000000000000000000000000000000000│ + d ▶│0000000000000000000000000000000000000000│ └────────────────────────────────────────┘ ▲───────────▲ - Slice + Shard -Fragment=intersection of frame & slice +Fragment=intersection of field & shard diff --git a/api.go b/api.go index 5b730b017..296304127 100644 --- a/api.go +++ b/api.go @@ -115,7 +115,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeRowAttrs: req.ExcludeRowAttrs, ExcludeColumns: req.ExcludeColumns, } - results, err := api.server.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { return resp, errors.Wrap(err, "executing") } @@ -316,21 +316,21 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str return nil } -// ExportCSV encodes the fragment designated by the index,field,slice as +// ExportCSV encodes the fragment designated by the index,field,shard as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } - // Validate that this handler owns the slice. - if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { - api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return ErrClusterDoesNotOwnSlice + // Validate that this handler owns the shard. + if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + return ErrClusterDoesNotOwnShard } // Find the fragment. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return ErrFragmentNotFound } @@ -354,25 +354,25 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return nil } -// SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { - if err := api.validate(apiSliceNodes); err != nil { +// ShardNodes returns the node and all replicas which should contain a shard's data. +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { + if err := api.validate(apiShardNodes); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.sliceNodes(indexName, slice), nil + return api.Cluster.shardNodes(indexName, shard), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return nil, ErrFragmentNotFound } @@ -382,7 +382,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName // UnmarshalFragment creates a new fragment (if necessary) and reads data from a // Reader which was previously written by MarshalFragment to populate the // fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error { +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { return errors.Wrap(err, "validating api method") } @@ -400,7 +400,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa } // Retrieve fragment from field. - frag, err := view.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -430,7 +430,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice) + f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Shard) if f == nil { return nil, ErrFragmentNotFound } @@ -448,13 +448,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, shard uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return nil, ErrFragmentNotFound } @@ -552,7 +552,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ignore this error because views do not exist on all nodes due to slice distribution. + // Ignore this error because views do not exist on all nodes due to shard distribution. if err != ErrInvalidView { return errors.Wrap(err, "deleting view") } @@ -641,13 +641,13 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s return attrs, nil } -// Import bulk imports data into a particular index,field,slice. +// Import bulk imports data into a particular index,field,shard. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } - _, field, err := api.indexField(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting field") } @@ -665,7 +665,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -676,21 +676,21 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return errors.Wrap(err, "validating api method") } - _, field, err := api.indexField(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting field") } // Import into fragment. err = field.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } -// MaxSlices returns the maximum slice number for each index in a map. -func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { - return api.Holder.MaxSlices() +// MaxShards returns the maximum shard number for each index in a map. +func (api *API) MaxShards(ctx context.Context) map[string]uint64 { + return api.Holder.MaxShards() } // StatsWithTags returns an instance of whatever implementation of StatsClient @@ -711,25 +711,25 @@ func (api *API) LongQueryTime() time.Duration { return api.Cluster.longQueryTime } -func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { - // Validate that this handler owns the slice. - if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { - api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return nil, nil, ErrClusterDoesNotOwnSlice +func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) { + // Validate that this handler owns the shard. + if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + return nil, nil, ErrClusterDoesNotOwnShard } // Find the Index. - api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, slice) + api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard) index := api.Holder.Index(indexName) if index == nil { - api.server.logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error()) + api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } // Retrieve field. field := index.Field(fieldName) if field == nil { - api.server.logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error()) + api.server.logger.Printf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error()) return nil, nil, ErrFieldNotFound } return index, field, nil @@ -851,12 +851,12 @@ func (api *API) Version() string { // Info returns information about this server instance func (api *API) Info() ServerInfo { return ServerInfo{ - SliceWidth: SliceWidth, + ShardWidth: ShardWidth, } } type ServerInfo struct { - SliceWidth uint64 `json:"sliceWidth"` + ShardWidth uint64 `json:"shardWidth"` } type apiMethod int @@ -881,14 +881,14 @@ const ( //apiLocalID // not implemented //apiLongQueryTime // not implemented apiMarshalFragment - //apiMaxSlices // not implemented + //apiMaxShards // not implemented apiQuery apiRecalculateCaches apiRemoveNode apiResizeAbort //apiSchema // not implemented apiSetCoordinator - apiSliceNodes + apiShardNodes //apiState // not implemented //apiStatsWithTags // not implemented apiUnmarshalFragment @@ -923,7 +923,7 @@ var methodsNormal = map[apiMethod]struct{}{ apiQuery: struct{}{}, apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, - apiSliceNodes: struct{}{}, + apiShardNodes: struct{}{}, apiUnmarshalFragment: struct{}{}, apiViews: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index a2b934e0f..7004e7258 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -2,15 +2,15 @@ package pilosa -import "fmt" +import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews" var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { - return fmt.Sprintf("apiMethod(%d)", i) + return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")" } return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] } diff --git a/broadcast.go b/broadcast.go index b7b13fe04..12f2b0bfe 100644 --- a/broadcast.go +++ b/broadcast.go @@ -62,7 +62,7 @@ type BroadcastHandler interface { // Broadcast message types. const ( - messageTypeCreateSlice = iota + messageTypeCreateShard = iota messageTypeCreateIndex messageTypeDeleteIndex messageTypeCreateField @@ -83,8 +83,8 @@ const ( func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { - case *internal.CreateSliceMessage: - typ = messageTypeCreateSlice + case *internal.CreateShardMessage: + typ = messageTypeCreateShard case *internal.CreateIndexMessage: typ = messageTypeCreateIndex case *internal.DeleteIndexMessage: @@ -129,8 +129,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { var m proto.Message switch typ { - case messageTypeCreateSlice: - m = &internal.CreateSliceMessage{} + case messageTypeCreateShard: + m = &internal.CreateShardMessage{} case messageTypeCreateIndex: m = &internal.CreateIndexMessage{} case messageTypeDeleteIndex: diff --git a/broadcast_test.go b/broadcast_test.go index 23bd962a7..5c97df0d0 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -30,9 +30,9 @@ import ( // Ensure a message can be marshaled and unmarshaled. func TestMessage_Marshal(t *testing.T) { - testMessageMarshal(t, &internal.CreateSliceMessage{ + testMessageMarshal(t, &internal.CreateShardMessage{ Index: "i", - Slice: 8, + Shard: 8, }) testMessageMarshal(t, &internal.DeleteIndexMessage{ diff --git a/client.go b/client.go index 292a71ef6..6a0b48d71 100644 --- a/client.go +++ b/client.go @@ -32,25 +32,25 @@ type FieldValue struct { // While I understand that putting the entire Client behind an interface might require this many methods, // I don't want to let it go unquestioned. type InternalClient interface { - MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) + MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) + FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error + Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error - ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error + ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error - FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) + FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error - RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) + RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) } //=============== @@ -81,7 +81,7 @@ func NewNopInternalClient() *NopInternalClient { var _ InternalClient = NewNopInternalClient() -func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (n *NopInternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { return nil, nil } func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { @@ -90,7 +90,7 @@ func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } -func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { +func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { @@ -99,7 +99,7 @@ func (n *NopInternalClient) Query(ctx context.Context, index string, queryReques func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { return nil, nil } -func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { +func (n *NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { return nil } func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { @@ -111,19 +111,19 @@ func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, option func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } -func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { +func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } -func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { +func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { return nil } func (n *NopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) { +func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) { return nil, nil } -func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { +func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { return nil, nil, nil } func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { @@ -135,6 +135,6 @@ func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fi func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { return nil } -func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { +func (n *NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index de728472a..21c91e528 100644 --- a/cluster.go +++ b/cluster.go @@ -576,7 +576,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool { type frag struct { field string view string - slice uint64 + shard uint64 } func fragsDiff(a, b []frag) []frag { @@ -623,15 +623,15 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { } } - return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews) + return c.fragCombos(idx.Name(), idx.MaxShard(), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index -// by creating every combination of field/view specified in `fieldViews` up to maxSlice. -func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { +// by creating every combination of field/view specified in `fieldViews` up to maxShard. +func (c *Cluster) fragCombos(idx string, maxShard uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) - for i := uint64(0); i <= maxSlice; i++ { - nodes := c.sliceNodes(idx, i) + for i := uint64(0); i <= maxShard; i++ { + nodes := c.shardNodes(idx, i) for _, n := range nodes { // for each field/view combination: for field, views := range fieldViews { @@ -762,7 +762,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R Index: idx.Name(), Field: frag.field, View: frag.view, - Slice: frag.slice, + Shard: frag.shard, } m[nodeID] = append(m[nodeID], src) @@ -772,10 +772,10 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// partition returns the partition that a slice belongs to. -func (c *Cluster) partition(index string, slice uint64) int { +// partition returns the partition that a shard belongs to. +func (c *Cluster) partition(index string, shard uint64) int { var buf [8]byte - binary.BigEndian.PutUint64(buf[:], slice) + binary.BigEndian.PutUint64(buf[:], shard) // Hash the bytes and mod by partition count. h := fnv.New64a() @@ -784,14 +784,14 @@ func (c *Cluster) partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.partitionN)) } -// sliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) sliceNodes(index string, slice uint64) []*Node { - return c.partitionNodes(c.partition(index, slice)) +// shardNodes returns a list of nodes that own a fragment. +func (c *Cluster) shardNodes(index string, shard uint64) []*Node { + return c.partitionNodes(c.partition(index, shard)) } -// ownsSlice returns true if a host owns a fragment. -func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID) +// ownsShard returns true if a host owns a fragment. +func (c *Cluster) ownsShard(nodeID string, index string, shard uint64) bool { + return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) } // partitionNodes returns a list of nodes that own a partition. @@ -817,20 +817,20 @@ func (c *Cluster) partitionNodes(partitionID int) []*Node { return nodes } -// containsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { +// containsShards is like OwnsShards, but it includes replicas. +func (c *Cluster) containsShards(index string, maxShard uint64, node *Node) []uint64 { + var shards []uint64 + for i := uint64(0); i <= maxShard; i++ { p := c.partition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { - slices = append(slices, i) + shards = append(shards, i) } } } - return slices + return shards } // Hasher represents an interface to hash integers into buckets. @@ -1211,7 +1211,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) srcURI := decodeURI(src.Node.URI) @@ -1228,27 +1228,27 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err } // Create the local fragment. - frag, err := v.CreateFragmentIfNotExists(src.Slice) + frag, err := v.CreateFragmentIfNotExists(src.Shard) if err != nil { return errors.Wrap(err, "creating fragment") } - // Stream slice from remote node. - c.logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) + // Stream shard from remote node. + c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) + rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found - // on the remote node. This occurs when a slice has been skipped and + // on the remote node. This occurs when a shard has been skipped and // therefore doesn't contain data. The coordinator correctly determined - // the resize instruction to retrieve the slice, but it doesn't have data. + // the resize instruction to retrieve the shard, but it doesn't have data. // TODO: figure out a way to distinguish from "fragment not found" errors // which are true errors and which simply mean the fragment doesn't have data. if err == ErrFragmentNotFound { return nil } - return errors.Wrap(err, "retrieving slice") + return errors.Wrap(err, "retrieving shard") } else if rd == nil { - return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) + return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI) } // Write to local field and always close reader. @@ -1257,7 +1257,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err _, err := frag.ReadFrom(rd) return err }(); err != nil { - return errors.Wrap(err, "copying remote slice") + return errors.Wrap(err, "copying remote shard") } } return nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index b53180b39..8e4621653 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -49,13 +49,13 @@ func TestFragCombos(t *testing.T) { tests := []struct { idx string - maxSlice uint64 + maxShard uint64 fieldViews viewsByField expected fragsByHost }{ { idx: "i", - maxSlice: uint64(2), + maxShard: uint64(2), fieldViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, @@ -64,7 +64,7 @@ func TestFragCombos(t *testing.T) { }, { idx: "foo", - maxSlice: uint64(3), + maxShard: uint64(3), fieldViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, @@ -74,7 +74,7 @@ func TestFragCombos(t *testing.T) { } for _, test := range tests { - actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews) + actual := c.fragCombos(test.idx, test.maxShard, test.fieldViews) if !reflect.DeepEqual(actual, test.expected) { t.Errorf("expected: %v, but got: %v", test.expected, actual) } @@ -339,13 +339,13 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { + if err := quick.Check(func(index string, shard uint64, partitionN int) bool { c := NewCluster() c.partitionN = partitionN - partitionID := c.partition(index, slice) + partitionID := c.partition(index, shard) if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) + t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) } return true @@ -380,14 +380,14 @@ func TestHasher(t *testing.T) { } } -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { +// Ensure ContainsShards can find the actual shard list for node and index. +func TestCluster_ContainsShards(t *testing.T) { c := NewTestCluster(5) c.ReplicaN = 3 - slices := c.containsSlices("test", 10, c.Nodes[2]) + shards := c.containsShards("test", 10, c.Nodes[2]) - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) + if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected shars for node's index: %v", shards) } } diff --git a/cmd/import.go b/cmd/import.go index 5a27b4055..4d565adb7 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Use: "import", Short: "Bulk load data into pilosa.", Long: `Bulk imports one or more CSV files to a host's index and field. The data -of the CSV file are grouped by slice for the most efficient import. +of the CSV file are grouped by shard for the most efficient import. The format of the CSV file is: diff --git a/ctl/export.go b/ctl/export.go index 090904807..5bfbd529f 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -80,16 +80,16 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return errors.Wrap(err, "creating client") } - // Determine slice count. - maxSlices, err := client.MaxSliceByIndex(ctx) + // Determine shard count. + maxShards, err := client.MaxShardByIndex(ctx) if err != nil { - return errors.Wrap(err, "getting slice count") + return errors.Wrap(err, "getting shard count") } - // Export each slice. - for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { - logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil { + // Export each shard. + for shard := uint64(0); shard <= maxShards[cmd.Index]; shard++ { + logger.Printf("exporting shard: %d", shard) + if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, shard, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/import.go b/ctl/import.go index 05659fa27..370425115 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -118,7 +118,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } - // Import each path and import by slice. + // Import each path and import by shard. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) if err := cmd.importPath(ctx, fieldType, path); err != nil { @@ -243,18 +243,18 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group bits by slice. + // Group bits by shard. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := http.Bits(bits).GroupBySlice() + bitsByShard := http.Bits(bits).GroupByShard() // Parse path into bits. - for slice, chunk := range bitsBySlice { + for shard, chunk := range bitsByShard { if cmd.Sort { sort.Sort(http.BitsByPos(chunk)) } - logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(chunk)) + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -437,18 +437,18 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group vals by slice. + // Group vals by shard. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := http.FieldValues(vals).GroupBySlice() + valsByShard := http.FieldValues(vals).GroupByShard() // Parse path into FieldValues. - for slice, vals := range valsBySlice { + for shard, vals := range valsByShard { if cmd.Sort { sort.Sort(http.FieldValues(vals)) } - logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(vals)) + if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/diagnostics.go b/diagnostics.go index b36335b38..74e7eebcf 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -216,14 +216,14 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { // EnrichWithSchemaProperties adds schema info to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { - var numSlices uint64 + var numShards uint64 numFields := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false for _, index := range d.server.holder.Indexes() { - numSlices += index.MaxSlice() + 1 + numShards += index.MaxShard() + 1 numIndexes += 1 for _, field := range index.Fields() { numFields += 1 @@ -238,7 +238,7 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { d.Set("NumIndexes", numIndexes) d.Set("NumFields", numFields) - d.Set("NumSlices", numSlices) + d.Set("NumShards", numShards) d.Set("BSIFieldCount", bsiFieldCount) d.Set("TimeQuantumEnabled", timeQuantumEnabled) } diff --git a/executor.go b/executor.go index 610696022..53d0de4a9 100644 --- a/executor.go +++ b/executor.go @@ -37,7 +37,7 @@ const ( rowLabel = "row" ) -// Executor recursively executes calls in a PQL query across all slices. +// Executor recursively executes calls in a PQL query across all shards. type Executor struct { Holder *Holder @@ -80,7 +80,7 @@ func NewExecutor(opts ...ExecutorOption) *Executor { } // Execute executes a PQL query. -func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { +func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { // Verify that an index is set. if index == "" { return nil, ErrIndexRequired @@ -108,7 +108,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } } - results, err := e.execute(ctx, index, q, slices, opt) + results, err := e.execute(ctx, index, q, shards, opt) if err != nil { return nil, err } @@ -123,24 +123,24 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return results, nil } -func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { - // Don't bother calculating slices for query types that don't require it. - needsSlices := needsSlices(q.Calls) +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { + // Don't bother calculating shards for query types that don't require it. + needsShards := needsShards(q.Calls) - // If slices are specified, then use that value for slices. If slices aren't + // If shards are specified, then use that value for shards. If shards aren't // specified, then include all of them. - if len(slices) == 0 && needsSlices { - // Round up the number of slices. + if len(shards) == 0 && needsShards { + // Round up the number of shards. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - maxSlice := idx.MaxSlice() + maxShard := idx.MaxShard() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) + // Generate a slice of all shards. + shards = make([]uint64, maxShard+1) + for i := range shards { + shards[i] = uint64(i) } } @@ -152,7 +152,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - v, err := e.executeCall(ctx, index, call, slices, opt) + v, err := e.executeCall(ctx, index, call, shards, opt) if err != nil { return nil, err } @@ -162,7 +162,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic } // executeCall executes a call. -func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { +func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } @@ -171,18 +171,18 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s switch c.Name { case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeSum(ctx, index, c, slices, opt) + return e.executeSum(ctx, index, c, shards, opt) case "Min": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeMin(ctx, index, c, slices, opt) + return e.executeMin(ctx, index, c, shards, opt) case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeMax(ctx, index, c, slices, opt) + return e.executeMax(ctx, index, c, shards, opt) case "Clear": return e.executeClearBit(ctx, index, c, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeCount(ctx, index, c, slices, opt) + return e.executeCount(ctx, index, c, shards, opt) case "Set": return e.executeSetBit(ctx, index, c, opt) case "SetValue": @@ -193,10 +193,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return nil, e.executeSetColumnAttrs(ctx, index, c, opt) case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeTopN(ctx, index, c, slices, opt) + return e.executeTopN(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeBitmapCall(ctx, index, c, slices, opt) + return e.executeBitmapCall(ctx, index, c, shards, opt) } } @@ -220,7 +220,7 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Sum(): field required") } @@ -230,8 +230,8 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeSumCountSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSumCountShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -240,7 +240,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other.Add(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -253,7 +253,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl } // executeMin executes a Min() call. -func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Min(): field required") } @@ -263,8 +263,8 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeMinSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMinShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -273,7 +273,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl return other.Smaller(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -286,7 +286,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl } // executeMax executes a Max() call. -func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Max(): field required") } @@ -296,8 +296,8 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeMaxSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMaxShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -306,7 +306,7 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl return other.Larger(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -319,10 +319,10 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Row, error) { +func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeBitmapCallSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeBitmapCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -335,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return other } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -384,31 +384,31 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return row, nil } -// executeBitmapCallSlice executes a bitmap call for a single slice. -func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBitmapCallShard executes a bitmap call for a single shard. +func (e *Executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { switch c.Name { case "Row": - return e.executeBitmapSlice(ctx, index, c, slice) + return e.executeBitmapShard(ctx, index, c, shard) case "Difference": - return e.executeDifferenceSlice(ctx, index, c, slice) + return e.executeDifferenceShard(ctx, index, c, shard) case "Intersect": - return e.executeIntersectSlice(ctx, index, c, slice) + return e.executeIntersectShard(ctx, index, c, shard) case "Range": - return e.executeRangeSlice(ctx, index, c, slice) + return e.executeRangeShard(ctx, index, c, shard) case "Union": - return e.executeUnionSlice(ctx, index, c, slice) + return e.executeUnionShard(ctx, index, c, shard) case "Xor": - return e.executeXorSlice(ctx, index, c, slice) + return e.executeXorShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } } -// executeSumCountSlice calculates the sum and count for bsiGroups on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeSumCountShard calculates the sum and count for bsiGroups on a shard. +func (e *Executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, errors.Wrap(err, "executing bitmap call") } @@ -427,7 +427,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -442,11 +442,11 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq }, nil } -// executeMinSlice calculates the min for bsiGroups on a slice. -func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMinShard calculates the min for bsiGroups on a shard. +func (e *Executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -465,7 +465,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -480,11 +480,11 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal }, nil } -// executeMaxSlice calculates the max for bsiGroups on a slice. -func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMaxShard calculates the max for bsiGroups on a shard. +func (e *Executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -503,7 +503,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -521,7 +521,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) { idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -532,7 +532,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s } // Execute original query. - pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) + pairs, err := e.executeTopNShards(ctx, index, c, shards, opt) if err != nil { return nil, errors.Wrap(err, "finding top results") } @@ -549,7 +549,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) + trimmedList, err := e.executeTopNShards(ctx, index, other, shards, opt) if err != nil { return nil, errors.Wrap(err, "retrieving full counts") } @@ -560,10 +560,10 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s return trimmedList, nil } -func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeTopNSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeTopNShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -572,7 +572,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return Pairs(other).Add(v.([]Pair)) } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -584,32 +584,32 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return results, nil } -// executeTopNSlice executes a TopN call for a single slice. -func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { +// executeTopNShard executes a TopN call for a single shard. +func (e *Executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { field, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } minThreshold, _, err := c.UintArg("threshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } attrValues, _ := c.Args["attrValues"].([]interface{}) tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } // Retrieve bitmap used to intersect. var src *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -623,7 +623,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca field = defaultField } - f := e.Holder.Fragment(index, field, ViewStandard, slice) + f := e.Holder.Fragment(index, field, ViewStandard, shard) if f == nil { return nil, nil } @@ -646,14 +646,14 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca }) } -// executeDifferenceSlice executes a difference() call for a local slice. -func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeDifferenceShard executes a difference() call for a local shard. +func (e *Executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Difference query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -668,7 +668,7 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c * return other, nil } -func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +func (e *Executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { @@ -693,21 +693,21 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice) + frag := e.Holder.Fragment(index, fieldName, ViewStandard, shard) if frag == nil { return NewRow(), nil } return frag.row(rowID), nil } -// executeIntersectSlice executes a intersect() call for a local slice. -func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeIntersectShard executes a intersect() call for a local shard. +func (e *Executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Intersect query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -722,11 +722,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p return other, nil } -// executeRangeSlice executes a range() call for a local slice. -func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeRangeShard executes a range() call for a local shard. +func (e *Executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Handle bsiGroup ranges differently. if c.HasConditionArg() { - return e.executeBSIGroupRangeSlice(ctx, index, c, slice) + return e.executeBSIGroupRangeShard(ctx, index, c, shard) } // Parse field. @@ -750,7 +750,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Read row & column id. rowID, rowOK, err := c.UintArg(fieldName) if err != nil { - return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) + return nil, fmt.Errorf("executeRangeShard - reading row: %v", err) } if !rowOK { return nil, fmt.Errorf("Range() must specify %q", rowLabel) @@ -785,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, fieldName, view, slice) + f := e.Holder.Fragment(index, fieldName, view, shard) if f == nil { continue } @@ -795,8 +795,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return row, nil } -// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice. -func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. +func (e *Executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Only one conditional should be present. if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") @@ -836,7 +836,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -857,7 +857,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // The reason we don't just call: // return f.RangeBetween(fieldName, predicates[0], predicates[1]) - // here is because we need the call to be slice-specific. + // here is because we need the call to be shard-specific. // Find bsiGroup. bsig := f.bsiGroup(fieldName) @@ -871,7 +871,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -904,7 +904,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -925,11 +925,11 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } } -// executeUnionSlice executes a union() call for a local slice. -func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeUnionShard executes a union() call for a local shard. +func (e *Executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -944,11 +944,11 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C return other, nil } -// executeXorSlice executes a xor() call for a local slice. -func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeXorShard executes a xor() call for a local shard. +func (e *Executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -964,7 +964,7 @@ func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Cal } // executeCount executes a count() call. -func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { +func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (uint64, error) { if len(c.Children) == 0 { return 0, errors.New("Count() requires an input bitmap") } else if len(c.Children) > 1 { @@ -972,8 +972,8 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + mapFn := func(shard uint64) (interface{}, error) { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err } @@ -986,7 +986,7 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, return other + v.(uint64) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return 0, err } @@ -1032,9 +1032,9 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // executeClearBitField executes a Clear() call for a single view. func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(rowID, colID, nil) @@ -1107,10 +1107,10 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, // executeSetBitField executes a Set() call for a specific view. func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(rowID, colID, timestamp) @@ -1389,12 +1389,12 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return nil } -// exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +// exec executes a PQL query remotely for a set of shards on a node. +func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ Query: q.String(), - Slices: slices, + Shards: shards, Remote: true, } @@ -1439,29 +1439,29 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * return results, nil } -// slicesByNode returns a mapping of nodes to slices. -// Returns errSliceUnavailable if a slice cannot be allocated to a node. -func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) { +// shardsByNode returns a mapping of nodes to shards. +// Returns errShardUnavailable if a shard cannot be allocated to a node. +func (e *Executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) { m := make(map[*Node][]uint64) loop: - for _, slice := range slices { - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, shard := range shards { + for _, node := range e.Cluster.shardNodes(index, shard) { if Nodes(nodes).Contains(node) { - m[node] = append(m[node], slice) + m[node] = append(m[node], shard) continue loop } } - return nil, errSliceUnavailable + return nil, errShardUnavailable } return m, nil } // mapReduce maps and reduces data across the cluster. // -// If a mapping of slices to a node fails then the slices are resplit across +// If a mapping of shards to a node fails then the shards are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. -func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { +func (e *Executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { ch := make(chan mapResponse) // Wrap context with a cancel to kill goroutines on exit. @@ -1480,13 +1480,13 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { return nil, errors.Wrap(err, "starting mapper") } // Iterate over all map responses and reduce. var result interface{} - var maxSlice int + var maxShard int for { select { case <-ctx.Done(): @@ -1500,7 +1500,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable { return nil, resp.err } else if err != nil { return nil, err @@ -1511,32 +1511,32 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, // Reduce value. result = reduceFn(result, resp.result) - // If all slices have been processed then return. - maxSlice += len(resp.slices) - if maxSlice >= len(slices) { + // If all shards have been processed then return. + maxShard += len(resp.shards) + if maxShard >= len(shards) { return result, nil } } } } -func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { - // Group slices together by nodes. - m, err := e.slicesByNode(nodes, index, slices) +func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { + // Group shards together by nodes. + m, err := e.shardsByNode(nodes, index, shards) if err != nil { return err } // Execute each node in a separate goroutine. - for n, nodeSlices := range m { - go func(n *Node, nodeSlices []uint64) { - resp := mapResponse{node: n, slices: nodeSlices} + for n, nodeShards := range m { + go func(n *Node, nodeShards []uint64) { + resp := mapResponse{node: n, shards: nodeShards} - // Send local slices to mapper, otherwise remote exec. + // Send local shards to mapper, otherwise remote exec. if n.ID == e.Node.ID { - resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) + resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, opt) if len(results) > 0 { resp.result = results[0] } @@ -1548,30 +1548,30 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod case <-ctx.Done(): case ch <- resp: } - }(n, nodeSlices) + }(n, nodeShards) } return nil } // mapperLocal performs map & reduce entirely on the local node. -func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { - ch := make(chan mapResponse, len(slices)) +func (e *Executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { + ch := make(chan mapResponse, len(shards)) - for _, slice := range slices { - go func(slice uint64) { - result, err := mapFn(slice) + for _, shard := range shards { + go func(shard uint64) { + result, err := mapFn(shard) // Return response to the channel. select { case <-ctx.Done(): case ch <- mapResponse{result: result, err: err}: } - }(slice) + }(shard) } // Reduce results - var maxSlice int + var maxShard int var result interface{} for { select { @@ -1582,11 +1582,11 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu return nil, resp.err } result = reduceFn(result, resp.result) - maxSlice++ + maxShard++ } - // Exit once all slices are processed. - if maxSlice == len(slices) { + // Exit once all shards are processed. + if maxShard == len(shards) { return result, nil } } @@ -1695,16 +1695,16 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res return result, nil } -// errSliceUnavailable is a marker error if no nodes are available. -var errSliceUnavailable = errors.New("slice unavailable") +// errShardUnavailable is a marker error if no nodes are available. +var errShardUnavailable = errors.New("shard unavailable") -type mapFunc func(slice uint64) (interface{}, error) +type mapFunc func(shard uint64) (interface{}, error) type reduceFunc func(prev, v interface{}) interface{} type mapResponse struct { node *Node - slices []uint64 + shards []uint64 result interface{} err error @@ -1740,7 +1740,7 @@ func hasOnlySetRowAttrs(calls []*pql.Call) bool { return true } -func needsSlices(calls []*pql.Call) bool { +func needsShards(calls []*pql.Call) bool { if len(calls) == 0 { return false } diff --git a/executor_test.go b/executor_test.go index 250d8b13d..dd1a714b8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -46,8 +46,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } @@ -57,7 +57,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) @@ -75,7 +75,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Inhibit row attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) @@ -95,12 +95,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } - if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } }) @@ -170,17 +170,17 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 1) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -201,16 +201,16 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -234,16 +234,16 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -253,8 +253,8 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "f", 10, 3) - hldr.SetBit("i", "f", 10, SliceWidth+1) - hldr.SetBit("i", "f", 10, SliceWidth+2) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { @@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { @@ -516,12 +516,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, f=0) Set(1, f=0) - Set(`+strconv.Itoa(SliceWidth)+`, f=0) - Set(`+strconv.Itoa(SliceWidth+2)+`, f=0) - Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0) + Set(`+strconv.Itoa(ShardWidth)+`, f=0) + Set(`+strconv.Itoa(ShardWidth+2)+`, f=0) + Set(`+strconv.Itoa((5*ShardWidth)+100)+`, f=0) Set(0, f=10) - Set(`+strconv.Itoa(SliceWidth)+`, f=10) - Set(`+strconv.Itoa(SliceWidth)+`, f=20) + Set(`+strconv.Itoa(ShardWidth)+`, f=10) + Set(`+strconv.Itoa(ShardWidth)+`, f=20) Set(0, other=0) `), nil, nil); err != nil { t.Fatal(err) @@ -546,7 +546,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { @@ -586,13 +586,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 0, 2) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 1, SliceWidth+2) - hldr.SetBit("i", "f", 1, SliceWidth) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 1, ShardWidth+2) + hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -611,22 +611,22 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 0, 2*SliceWidth) - hldr.SetBit("i", "f", 0, 3*SliceWidth) - hldr.SetBit("i", "f", 0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 0, 2*ShardWidth) + hldr.SetBit("i", "f", 0, 3*ShardWidth) + hldr.SetBit("i", "f", 0, 4*ShardWidth) hldr.SetBit("i", "f", 1, 0) hldr.SetBit("i", "f", 1, 1) - hldr.SetBit("i", "f", 2, SliceWidth) - hldr.SetBit("i", "f", 2, SliceWidth+1) + hldr.SetBit("i", "f", 2, ShardWidth) + hldr.SetBit("i", "f", 2, ShardWidth+1) - hldr.SetBit("i", "f", 3, 2*SliceWidth) - hldr.SetBit("i", "f", 3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*ShardWidth) + hldr.SetBit("i", "f", 3, 2*ShardWidth+1) - hldr.SetBit("i", "f", 4, 3*SliceWidth) - hldr.SetBit("i", "f", 4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*ShardWidth) + hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -644,20 +644,20 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 10, SliceWidth) - hldr.SetBit("i", "f", 10, SliceWidth+1) - hldr.SetBit("i", "f", 20, SliceWidth) - hldr.SetBit("i", "f", 20, SliceWidth+1) - hldr.SetBit("i", "f", 20, SliceWidth+2) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth) + hldr.SetBit("i", "f", 20, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth+2) // Create an intersecting row. - hldr.SetBit("i", "other", 100, SliceWidth) - hldr.SetBit("i", "other", 100, SliceWidth+1) - hldr.SetBit("i", "other", 100, SliceWidth+2) + hldr.SetBit("i", "other", 100, ShardWidth) + hldr.SetBit("i", "other", 100, ShardWidth+1) + hldr.SetBit("i", "other", 100, ShardWidth+2) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() @@ -683,7 +683,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, ShardWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) @@ -706,7 +706,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, ShardWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -747,18 +747,18 @@ func TestExecutor_Execute_MinMax(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, x=0) Set(3, x=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) Set(1, x=1) - Set(`+strconv.Itoa(SliceWidth+2)+`, x=2) + Set(`+strconv.Itoa(ShardWidth+2)+`, x=2) SetValue(col=0, f=20) SetValue(col=1, f=-5) SetValue(col=2, f=-5) SetValue(col=3, f=10) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, f=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, f=40) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, f=50) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, f=60) `), nil, nil); err != nil { t.Fatal(err) } @@ -857,14 +857,14 @@ func TestExecutor_Execute_Sum(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, x=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) SetValue(col=0, foo=20) SetValue(col=0, bar=2000) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=40) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=50) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) SetValue(col=0, other=1000) `), nil, nil); err != nil { t.Fatal(err) @@ -979,14 +979,14 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, f=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, f=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, f=0) SetValue(col=50, foo=20) SetValue(col=50, bar=2000) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=10) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=20) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) SetValue(col=0, other=1000) SetValue(col=0, edge=100) SetValue(col=1, edge=-100) @@ -997,7 +997,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // NEQ if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - @@ -1027,7 +1027,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("LT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1035,7 +1035,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1043,7 +1043,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("GT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1051,7 +1051,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("GTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1129,35 +1129,35 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Row(f=10)` { t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{1}) { - t.Fatalf("unexpected slices: %+v", slices) + } else if !reflect.DeepEqual(shards, []uint64{1}) { + t.Fatalf("unexpected shards: %+v", shards) } - // Set columns in slice 0 & 2. + // Set columns in shard 0 & 2. r := pilosa.NewRow( - (0*SliceWidth)+1, - (0*SliceWidth)+2, - (2*SliceWidth)+4, + (0*ShardWidth)+1, + (0*ShardWidth)+2, + (2*ShardWidth)+4, ) return []interface{}{r}, nil } // Create local executor data. - // The local node owns slice 1. + // The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*ShardWidth + 4}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -1180,16 +1180,16 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(10)}, nil } - // Create local executor data. The local node owns slice 1. + // Create local executor data. The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, (2*SliceWidth)+1) - hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) + hldr.SetBit("i", "f", 10, (2*ShardWidth)+1) + hldr.SetBit("i", "f", 10, (2*ShardWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { @@ -1219,7 +1219,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Set(_col=2, f=10)` { @@ -1274,7 +1274,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` { @@ -1330,15 +1330,15 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if !reflect.DeepEqual(slices, []uint64{1, 3}) { - t.Fatalf("unexpected slices: %+v", slices) + } else if !reflect.DeepEqual(shards, []uint64{1, 3}) { + t.Fatalf("unexpected shards: %+v", shards) } // Query should be executed twice. Once to get the top bitmaps for the - // slices and a second time to get the counts for a set of bitmaps. + // shards and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: if query.String() != `TopN(_field="f", n=3)` { @@ -1361,12 +1361,12 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { }}, nil } - // Create local executor data on slice 2 & 4. + // Create local executor data on shard 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 30, (2*SliceWidth)+1) - hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) + hldr.SetBit("i", "f", 30, (2*ShardWidth)+1) + hldr.SetBit("i", "f", 30, (4*ShardWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil { @@ -1396,7 +1396,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) } else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` { @@ -1407,7 +1407,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { } // Create local executor data. - // The local node owns slice 1. + // The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() @@ -1417,7 +1417,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { } f := hldr.Field("i", "f") s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+1) e := test.NewExecutor(hldr.Holder, c) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { diff --git a/field.go b/field.go index 4fd684682..23de79f7c 100644 --- a/field.go +++ b/field.go @@ -152,15 +152,15 @@ func (f *Field) Path() string { return f.path } // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } -// MaxSlice returns the max slice in the field. -func (f *Field) MaxSlice() uint64 { +// MaxShard returns the max shard in the field. +func (f *Field) MaxShard() uint64 { f.mu.RLock() defer f.mu.RUnlock() var max uint64 for _, view := range f.views { - if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max { - max = viewMaxSlice + if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max { + max = viewMaxShard } } return max @@ -665,7 +665,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) { return view.row(rowID), nil } -// ViewRow returns a row for a view and slice. +// ViewRow returns a row for a view and shard. // TODO: unexport this with views (it's only used in tests). func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { view := f.View(viewName) @@ -934,7 +934,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Attach bit to each standard view. for _, name := range standard { - key := importKey{View: name, Slice: columnID / SliceWidth} + key := importKey{View: name, Shard: columnID / ShardWidth} data := dataByFragment[key] data.RowIDs = append(data.RowIDs, rowID) data.ColumnIDs = append(data.ColumnIDs, columnID) @@ -949,7 +949,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.Wrap(err, "creating view") } - frag, err := view.CreateFragmentIfNotExists(key.Slice) + frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { return errors.Wrap(err, "creating view") } @@ -983,7 +983,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { // Attach value to each bsiGroup view. for _, name := range []string{viewName} { - key := importKey{View: name, Slice: columnID / SliceWidth} + key := importKey{View: name, Shard: columnID / ShardWidth} data := dataByFragment[key] data.ColumnIDs = append(data.ColumnIDs, columnID) data.Values = append(data.Values, value) @@ -1001,7 +1001,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { return errors.Wrap(err, "creating view") } - frag, err := view.CreateFragmentIfNotExists(key.Slice) + frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -1192,7 +1192,7 @@ func (b *bsiGroup) BitDepth() uint { // Note that in this case (because the range uses the full BitDepth 0 to 1023), // we can't simply return 1024. // In order to make this work, we effectively need to change the operator to LTE. -// Executor.executeBSIGroupRangeSlice() takes this into account and returns +// Executor.executeBSIGroupRangeShard() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { diff --git a/fragment.go b/fragment.go index 9b4780795..891c1570e 100644 --- a/fragment.go +++ b/fragment.go @@ -44,8 +44,8 @@ import ( ) const ( - // SliceWidth is the number of column IDs in a slice. - SliceWidth = 1048576 + // ShardWidth is the number of column IDs in a shard. + ShardWidth = 1048576 // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" @@ -63,7 +63,7 @@ const ( defaultFragmentMaxOpN = 2000 ) -// Fragment represents the intersection of a field and slice in an index. +// Fragment represents the intersection of a field and shard in an index. type Fragment struct { mu sync.RWMutex @@ -71,7 +71,7 @@ type Fragment struct { index string field string view string - slice uint64 + shard uint64 // File-backed storage path string @@ -110,13 +110,13 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, index, field, view string, slice uint64) *Fragment { +func NewFragment(path, index, field, view string, shard uint64) *Fragment { return &Fragment{ path: path, index: index, field: field, view: view, - slice: slice, + shard: shard, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -151,7 +151,7 @@ func (f *Fragment) Open() error { // Read last bit to determine max row. pos := f.storage.Max() - f.maxRowID = pos / SliceWidth + f.maxRowID = pos / ShardWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) return nil @@ -165,7 +165,7 @@ func (f *Fragment) Open() error { // openStorage opens the storage bitmap. func (f *Fragment) openStorage() error { - // Create a roaring bitmap to serve as storage for the slice. + // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() } @@ -257,7 +257,7 @@ func (f *Fragment) openCache() error { // Read in all rows by ID. // This will cause them to be added to the cache. for _, id := range pb.IDs { - n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) + n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth) f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -337,7 +337,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by - data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth) + data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. // We Clone() data because otherwise row will contains pointers to containers in storage. @@ -345,7 +345,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac row := &Row{ segments: []RowSegment{{ data: *data.Clone(), - slice: f.slice, + shard: f.shard, writable: false, }}, } @@ -837,9 +837,9 @@ func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64 // pos translates the row ID and column ID into a position in the storage bitmap. func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { - // Return an error if the column ID is out of the range of the fragment's slice. - minColumnID := f.slice * SliceWidth - if columnID < minColumnID || columnID >= minColumnID+SliceWidth { + // Return an error if the column ID is out of the range of the fragment's shard. + minColumnID := f.shard * ShardWidth + if columnID < minColumnID || columnID >= minColumnID+ShardWidth { return 0, errors.New("column out of bounds") } return pos(rowID, columnID), nil @@ -859,7 +859,7 @@ func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error { } // Invoke caller's function. - err = fn(i/SliceWidth, (f.slice*SliceWidth)+(i%SliceWidth)) + err = fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)) }) return err } @@ -1093,16 +1093,16 @@ func (f *Fragment) Blocks() []FragmentBlock { if eof { return nil } - blockID := int(v / (HashBlockSize * SliceWidth)) + blockID := int(v / (HashBlockSize * ShardWidth)) for { // Check for multiple block checksums in a row. if n := f.readContiguousChecksums(&a, blockID); n > 0 { - itr.Seek(uint64(blockID+n) * HashBlockSize * SliceWidth) + itr.Seek(uint64(blockID+n) * HashBlockSize * ShardWidth) v, eof = itr.Next() if eof { break } - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) continue } @@ -1113,7 +1113,7 @@ func (f *Fragment) Blocks() []FragmentBlock { // Read all values for the block. for ; ; v, eof = itr.Next() { // Once we hit the next block, save the value for the next iteration. - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) if blockID != h.blockID || eof { break } @@ -1160,9 +1160,9 @@ func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() - f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) { - rowIDs = append(rowIDs, i/SliceWidth) - columnIDs = append(columnIDs, i%SliceWidth) + f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) { + rowIDs = append(rowIDs, i/ShardWidth) + columnIDs = append(columnIDs, i%ShardWidth) }) return } @@ -1190,7 +1190,7 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize - maxColumnID := uint64(SliceWidth) + maxColumnID := uint64(ShardWidth) // Create buffered iterator for local block. itrs := make([]*BufIterator, 1, len(data)+1) @@ -1278,14 +1278,14 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Set local bits. for i := range sets[0].columnIDs { - if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil { + if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.shard*ShardWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. for i := range clears[0].columnIDs { - if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.shard*ShardWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1423,8 +1423,8 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *Fragment) snapshot() error { - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice) - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) + completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) @@ -1736,7 +1736,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice) + nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard) if len(nodes) == 1 { return nil } @@ -1752,7 +1752,7 @@ func (s *FragmentSyncer) syncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1817,7 +1817,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var uris []*URI var pairSets []pairSet - for _, node := range s.Cluster.sliceNodes(f.index, f.slice) { + for _, node := range s.Cluster.shardNodes(f.index, f.shard) { if s.Node.ID == node.ID { continue } @@ -1831,7 +1831,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.shard, id) if err != nil { return errors.Wrap(err, "getting block") } @@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) count++ } for j := 0; j < len(clear.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) count++ } @@ -1933,5 +1933,5 @@ func byteSlicesEqual(a [][]byte) bool { // pos returns the row position of a row/column pair. func pos(rowID, columnID uint64) uint64 { - return (rowID * SliceWidth) + (columnID % SliceWidth) + return (rowID * ShardWidth) + (columnID % ShardWidth) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e7a77dafb..c779dd596 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -742,7 +742,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { - slice := uint64(0) + shard := uint64(0) cacheSize := uint32(3) // Create Index. @@ -762,7 +762,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Create fragment. - frag, err := view.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } @@ -1181,7 +1181,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { for row := 0; row < 100; row++ { val := 1 i := 0 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 @@ -1215,7 +1215,7 @@ func BenchmarkFragment_Import(b *testing.B) { i := 0 for row := 0; row < 100; row++ { val := 1 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 @@ -1237,7 +1237,7 @@ func BenchmarkFragment_Import(b *testing.B) { ///////////////////////////////////////////////////////////////////// // mustOpenFragment returns a new instance of Fragment with a temporary path. -func mustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { +func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -1248,7 +1248,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string) cacheType = DefaultCacheType } - f := NewFragment(file.Name(), index, field, view, slice) + f := NewFragment(file.Name(), index, field, view, shard) f.CacheType = cacheType f.RowAttrStore = newMemAttrStore() diff --git a/handler.go b/handler.go index c9a476e13..1f3d04300 100644 --- a/handler.go +++ b/handler.go @@ -12,9 +12,9 @@ type QueryRequest struct { // The query string to parse and execute. Query string - // The slices to include in the query execution. - // If empty, all slices are included. - Slices []uint64 + // The shards to include in the query execution. + // If empty, all shards are included. + Shards []uint64 // Return column attributes, if true. ColumnAttrs bool diff --git a/holder.go b/holder.go index 316ca1244..bfb2ae758 100644 --- a/holder.go +++ b/holder.go @@ -200,11 +200,11 @@ func (h *Holder) HasData() (bool, error) { return false, nil } -// MaxSlices returns MaxSlice map for all indexes. -func (h *Holder) MaxSlices() map[string]uint64 { +// MaxShards returns MaxShard map for all indexes. +func (h *Holder) MaxShards() map[string]uint64 { a := make(map[string]uint64) for _, index := range h.Indexes() { - a[index.Name()] = index.MaxSlice() + a[index.Name()] = index.MaxShard() } return a } @@ -257,10 +257,10 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { return nil } -// EncodeMaxSlices creates and internal representation of max slices. -func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { - return &internal.MaxSlices{ - Standard: h.MaxSlices(), +// EncodeMaxShards creates and internal representation of max shards. +func (h *Holder) EncodeMaxShards() *internal.MaxShards { + return &internal.MaxShards{ + Standard: h.MaxShards(), } } @@ -411,13 +411,13 @@ func (h *Holder) View(index, field, name string) *View { return f.View(name) } -// Fragment returns the fragment for an index, field & slice. -func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment { +// Fragment returns the fragment for an index, field & shard. +func (h *Holder) Fragment(index, field, view string, shard uint64) *Fragment { v := h.View(index, field, view) if v == nil { return nil } - return v.Fragment(slice) + return v.Fragment(shard) } // monitorCacheFlush periodically flushes all fragment caches sequentially. @@ -619,9 +619,9 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { - // Ignore slices that this host doesn't own. - if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) { + for shard := uint64(0); shard <= s.Holder.Index(di.Name).MaxShard(); shard++ { + // Ignore shards that this host doesn't own. + if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) { continue } @@ -631,8 +631,8 @@ func (s *HolderSyncer) SyncHolder() error { } // Sync fragment if own it. - if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + if err := s.syncFragment(di.Name, fi.Name, vi.Name, shard); err != nil { + return fmt.Errorf("fragment sync error: index=%s, field=%s, shard=%d, err=%s", di.Name, fi.Name, shard, err) } } } @@ -736,7 +736,7 @@ func (s *HolderSyncer) syncField(index, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error { +func (s *HolderSyncer) syncFragment(index, field, view string, shard uint64) error { // Retrieve local field. f := s.Holder.Field(index, field) if f == nil { @@ -750,7 +750,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err } // Ensure fragment exists locally. - frag, err := v.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -800,19 +800,19 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node) + containedShards := c.Cluster.containsShards(index.Name(), index.MaxShard(), c.Node) // Get the fragments registered in memory. for _, field := range index.Fields() { for _, view := range field.Views() { for _, fragment := range view.allFragments() { - fragSlice := fragment.slice + fragShard := fragment.shard // Ignore fragments that should be present. - if uint64InSlice(fragSlice, containedSlices) { + if uint64InSlice(fragShard, containedShards) { continue } // Delete fragment. - if err := view.deleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragShard); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_test.go b/holder_test.go index 9b3e21c12..f758798a4 100644 --- a/holder_test.go +++ b/holder_test.go @@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=opening storage: unmarshal storage") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") { t.Fatalf("unexpected error: %s", err) } }) @@ -373,12 +373,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1 := test.MustOpenHolder() defer hldr1.Close() s.Handler.API.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } // Mock 2-node, fully replicated cluster. @@ -400,7 +400,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.SetBit("i", "f", 120, 10) hldr0.SetBit("i", "f", 200, 4) - hldr0.SetBit("i", "f0", 9, SliceWidth+5) + hldr0.SetBit("i", "f0", 9, ShardWidth+5) // Set a bit to create the fragment. hldr0.SetBit("y", "z", 0, 0) @@ -410,13 +410,13 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1.SetBit("i", "f", 3, 10) hldr1.SetBit("i", "f", 120, 10) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7) - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(3) + // Set highest shard. + hldr0.Index("i").SetRemoteMaxShard(1) + hldr0.Index("y").SetRemoteMaxShard(3) // Set up syncer. syncer := pilosa.HolderSyncer{ @@ -444,11 +444,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -482,15 +482,15 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { hldr0.SetBit("i", "f", 120, 10) hldr0.SetBit("i", "f", 200, 4) - hldr0.SetBit("i", "f0", 9, SliceWidth+5) + hldr0.SetBit("i", "f0", 9, ShardWidth+5) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(2) + // Set highest shard. + hldr0.Index("i").SetRemoteMaxShard(1) + hldr0.Index("y").SetRemoteMaxShard(2) // Keep replication the same and ensure we get the expected results. cluster.ReplicaN = 2 @@ -520,11 +520,11 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -562,7 +562,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/http/client.go b/http/client.go index 378413c2f..59efb4f72 100644 --- a/http/client.go +++ b/http/client.go @@ -73,15 +73,15 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) // Host returns the host the client was initialized with. func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) +// MaxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxShardByIndex(ctx) } -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +// maxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") + u := uriPathToURL(c.defaultURI, "/shards/max") // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -99,7 +99,7 @@ func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64 } defer resp.Body.Close() - var rsp getSlicesMaxResponse + var rsp getShardsMaxResponse if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("http: status=%d", resp.StatusCode) } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { @@ -184,11 +184,11 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo } } -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { +// FragmentNodes returns a list of nodes that own a shard. +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() + u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode() // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -272,23 +272,23 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s return qresp, nil } -// Import bulk imports bits for a single slice to a host. -func (c *InternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { +// Import bulk imports bits for a single shard to a host. +func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - buf, err := marshalImportPayload(index, field, slice, bits) + buf, err := marshalImportPayload(index, field, shard, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Import to each node. @@ -343,7 +343,7 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) { +func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() @@ -353,7 +353,7 @@ func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, RowIDs: rowIDs, ColumnIDs: columnIDs, Timestamps: timestamps, @@ -423,23 +423,23 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, buf return nil } -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { +// ImportValue bulk imports field values for a single shard to a host. +func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - buf, err := marshalImportValuePayload(index, field, slice, vals) + buf, err := marshalImportValuePayload(index, field, shard, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Import to each node. @@ -453,7 +453,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) { +func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() @@ -462,7 +462,7 @@ func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, ColumnIDs: columnIDs, Values: values, }) @@ -510,18 +510,18 @@ func (c *InternalClient) importValueNode(ctx context.Context, node *pilosa.Node, return nil } -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { +// ExportCSV bulk exports data for a single shard from a host to CSV format. +func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Attempt nodes in random order. @@ -529,7 +529,7 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, field, shard, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { @@ -541,13 +541,13 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli } // exportNode copies a CSV export from a node to w. -func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Generate HTTP request. @@ -578,19 +578,19 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i return nil } -func (c *InternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { node := &pilosa.Node{ URI: uri, } - return c.backupSliceNode(ctx, index, field, slice, node) + return c.backupShardNode(ctx, index, field, shard, node) } -func (c *InternalClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { +func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Build request. @@ -671,7 +671,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64) ([]pilosa.FragmentBlock, error) { if uri == nil { uri = c.defaultURI } @@ -679,7 +679,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Build request. @@ -716,11 +716,11 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in } // BlockData returns row/column id pairs for a block. -func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, Block: uint64(block), }) if err != nil { @@ -952,17 +952,17 @@ func (p Bits) Timestamps() []int64 { return other } -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit { +// GroupByShard returns a map of bits by shard. +func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { m := make(map[uint64][]pilosa.Bit) for _, bit := range p { - slice := bit.ColumnID / pilosa.SliceWidth - m[slice] = append(m[slice], bit) + shard := bit.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], bit) } - for slice, bits := range m { + for shard, bits := range m { sort.Sort(Bits(bits)) - m[slice] = bits + m[shard] = bits } return m @@ -996,17 +996,17 @@ func (p FieldValues) Values() []int64 { return other } -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue { +// GroupByShard returns a map of field values by shard. +func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { m := make(map[uint64][]pilosa.FieldValue) for _, val := range p { - slice := val.ColumnID / pilosa.SliceWidth - m[slice] = append(m[slice], val) + shard := val.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], val) } - for slice, vals := range m { + for shard, vals := range m { sort.Sort(FieldValues(vals)) - m[slice] = vals + m[shard] = vals } return m @@ -1027,7 +1027,7 @@ func (p BitsByPos) Less(i, j int) bool { // pos returns the row position of a row/column pair. func pos(rowID, columnID uint64) uint64 { - return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth) + return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) } func uriPathToURL(uri *pilosa.URI, path string) url.URL { diff --git a/http/client_test.go b/http/client_test.go index 7363879ce..e5a595605 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -62,42 +62,42 @@ func TestClient_MultiNode(t *testing.T) { defer s[i].Close() } - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - sliceNums := []uint64{1, 2, 6} + // Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN. + shardNums := []uint64{1, 2, 6} - // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())` + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())` owns := [][]uint64{ {1, 3, 4, 8, 10, 13, 17, 19}, {2, 5, 7, 11, 12, 14, 18}, {0, 6, 9, 15, 16, 20}, } - for i, num := range sliceNums { + for i, num := range shardNums { ownsNum := false for _, ownNum := range owns[i] { if ownNum == num { @@ -106,18 +106,18 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, s[i].Host(), owns) } } - baseBit0 := pilosa.SliceWidth * sliceNums[0] - baseBit1 := pilosa.SliceWidth * sliceNums[1] - baseBit2 := pilosa.SliceWidth * sliceNums[2] + baseBit0 := pilosa.ShardWidth * shardNums[0] + baseBit1 := pilosa.ShardWidth * shardNums[1] + baseBit2 := pilosa.ShardWidth * shardNums[2] - maxSlice := uint64(0) - for _, x := range sliceNums { - if x > maxSlice { - maxSlice = x + maxShard := uint64(0) + for _, x := range shardNums { + if x > maxShard { + maxShard = x } } @@ -145,9 +145,9 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[0]).RecalculateCache() + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[1]).RecalculateCache() + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[2]).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -165,18 +165,18 @@ func TestClient_MultiNode(t *testing.T) { t.Fatal(err) } - // Check the results before every node has the correct max slice value. + // Check the results before every node has the correct max shard value. pairs := result.Results[0].Pairs for _, pair := range pairs { if pair.ID == 22 && pair.Count != 3 { - t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) + t.Fatalf("Invalid Cluster wide MaxShard prevents accurate calculation of %s", pair) } } - // Set max slice to correct value. - hldr[0].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) + // Set max shard to correct value. + hldr[0].Index("i").SetRemoteMaxShard(maxShard) + hldr[1].Index("i").SetRemoteMaxShard(maxShard) + hldr[2].Index("i").SetRemoteMaxShard(maxShard) result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { @@ -328,7 +328,7 @@ func TestClient_FragmentBlocks(t *testing.T) { hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) - // Set a bit on a different slice. + // Set a bit on a different shard. hldr.SetBit("i", "f", 0, 1) c := MustNewClient(cmd.URL(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) diff --git a/http/handler.go b/http/handler.go index 58a3f751d..4cc7099ba 100644 --- a/http/handler.go +++ b/http/handler.go @@ -156,13 +156,13 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") + h.validators["GetShardMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "shard") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -195,7 +195,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") @@ -385,20 +385,20 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { +// handleGetShardsMax handles GET /shards/max requests. +func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), + if err := json.NewEncoder(w).Encode(getShardsMaxResponse{ + Standard: h.API.MaxShards(r.Context()), }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) + h.Logger.Printf("write shards-max response error: %s", err) } } -type getSlicesMaxResponse struct { +type getShardsMaxResponse struct { Standard map[string]uint64 `json:"standard"` } @@ -839,15 +839,15 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } query := string(buf) - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) + // Parse list of shards. + shards, err := parseUint64Slice(q.Get("shards")) if err != nil { - return nil, errors.New("invalid slice argument") + return nil, errors.New("invalid shard argument") } return &pilosa.QueryRequest{ Query: query, - Slices: slices, + Shards: shards, ColumnAttrs: q.Get("columnAttrs") == "true", ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", ExcludeColumns: q.Get("excludeColumns") == "true", @@ -908,7 +908,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { fallthrough case pilosa.ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -961,7 +961,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) fallthrough case pilosa.ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -998,17 +998,17 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() index, field := q.Get("index"), q.Get("field") - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) + http.Error(w, "invalid shard", http.StatusBadRequest) return } - if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { + if err = h.API.ExportCSV(r.Context(), index, field, shard, w); err != nil { switch errors.Cause(err) { case pilosa.ErrFragmentNotFound: break - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1026,15 +1026,15 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) q := r.URL.Query() index := q.Get("index") - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + // Read shard parameter. + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) + http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) return } // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) + nodes, err := h.API.ShardNodes(r.Context(), index, shard) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1072,15 +1072,15 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - // Read slice parameter. + // Read shard parameter. q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) + http.Error(w, "shard required", http.StatusBadRequest) return } - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard) if err != nil { if errors.Cause(err) == pilosa.ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -1131,7 +1131,7 @@ const ( func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { req := &pilosa.QueryRequest{ Query: pb.Query, - Slices: pb.Slices, + Shards: pb.Shards, ColumnAttrs: pb.ColumnAttrs, Remote: pb.Remote, ExcludeRowAttrs: pb.ExcludeRowAttrs, diff --git a/index.go b/index.go index a118b1808..ae6141b90 100644 --- a/index.go +++ b/index.go @@ -38,8 +38,8 @@ type Index struct { // Fields by name. fields map[string]*Field - // Max Slice on any node in the cluster, according to this node. - remoteMaxSlice uint64 + // Max shard on any node in the cluster, according to this node. + remoteMaxShard uint64 NewAttrStore func(string) AttrStore @@ -64,7 +64,7 @@ func NewIndex(path, name string) (*Index, error) { name: name, fields: make(map[string]*Field), - remoteMaxSlice: 0, + remoteMaxShard: 0, NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, @@ -210,30 +210,30 @@ func (i *Index) Close() error { return nil } -// MaxSlice returns the max slice in the index according to this node. -func (i *Index) MaxSlice() uint64 { +// MaxShard returns the max shard in the index according to this node. +func (i *Index) MaxShard() uint64 { if i == nil { return 0 } i.mu.RLock() defer i.mu.RUnlock() - max := i.remoteMaxSlice + max := i.remoteMaxShard for _, f := range i.fields { - if slice := f.MaxSlice(); slice > max { - max = slice + if shard := f.MaxShard(); shard > max { + max = shard } } - i.Stats.Gauge("maxSlice", float64(max), 1.0) + i.Stats.Gauge("maxShard", float64(max), 1.0) return max } -// SetRemoteMaxSlice sets the remote max slice value received from another node. -func (i *Index) SetRemoteMaxSlice(newmax uint64) { +// SetRemoteMaxShard sets the remote max shard value received from another node. +func (i *Index) SetRemoteMaxShard(newmax uint64) { i.mu.Lock() defer i.mu.Unlock() - i.remoteMaxSlice = newmax + i.remoteMaxShard = newmax } // FieldPath returns the path to a field in the index. @@ -427,7 +427,7 @@ func hasTime(a []*time.Time) bool { type importKey struct { View string - Slice uint64 + Shard uint64 } type importData struct { diff --git a/index_test.go b/index_test.go index 5beb67f00..d1a740a4f 100644 --- a/index_test.go +++ b/index_test.go @@ -23,8 +23,8 @@ import ( "github.com/pilosa/pilosa/test" ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth +// ShardWidth is a helper reference to use when testing. +const ShardWidth = pilosa.ShardWidth // Ensure index can open and retrieve a field. func TestIndex_CreateFieldIfNotExists(t *testing.T) { diff --git a/internal/private.pb.go b/internal/private.pb.go index c3dadb455..8fdcfa7db 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: private.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -14,8 +15,8 @@ BlockDataRequest BlockDataResponse Cache - MaxSlices - CreateSliceMessage + MaxShards + CreateShardMessage DeleteIndexMessage CreateIndexMessage CreateFieldMessage @@ -159,7 +160,7 @@ type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } @@ -189,9 +190,9 @@ func (m *BlockDataRequest) GetView() string { return "" } -func (m *BlockDataRequest) GetSlice() uint64 { +func (m *BlockDataRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -243,42 +244,42 @@ func (m *Cache) GetIDs() []uint64 { return nil } -type MaxSlices struct { +type MaxShards struct { Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxSlices) Reset() { *m = MaxSlices{} } -func (m *MaxSlices) String() string { return proto.CompactTextString(m) } -func (*MaxSlices) ProtoMessage() {} -func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } +func (m *MaxShards) Reset() { *m = MaxShards{} } +func (m *MaxShards) String() string { return proto.CompactTextString(m) } +func (*MaxShards) ProtoMessage() {} +func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } -func (m *MaxSlices) GetStandard() map[string]uint64 { +func (m *MaxShards) GetStandard() map[string]uint64 { if m != nil { return m.Standard } return nil } -type CreateSliceMessage struct { +type CreateShardMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } -func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) } -func (*CreateSliceMessage) ProtoMessage() {} -func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } +func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } +func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } +func (*CreateShardMessage) ProtoMessage() {} +func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } -func (m *CreateSliceMessage) GetIndex() string { +func (m *CreateShardMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *CreateSliceMessage) GetSlice() uint64 { +func (m *CreateShardMessage) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -565,7 +566,7 @@ func (m *NodeEventMessage) GetNode() *Node { type NodeStatus struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` + MaxShards *MaxShards `protobuf:"bytes,2,opt,name=MaxShards" json:"MaxShards,omitempty"` Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` } @@ -581,9 +582,9 @@ func (m *NodeStatus) GetNode() *Node { return nil } -func (m *NodeStatus) GetMaxSlices() *MaxSlices { +func (m *NodeStatus) GetMaxShards() *MaxShards { if m != nil { - return m.MaxSlices + return m.MaxShards } return nil } @@ -792,7 +793,7 @@ type ResizeSource struct { Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } func (m *ResizeSource) Reset() { *m = ResizeSource{} } @@ -828,9 +829,9 @@ func (m *ResizeSource) GetView() string { return "" } -func (m *ResizeSource) GetSlice() uint64 { +func (m *ResizeSource) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -940,8 +941,8 @@ func init() { proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices") - proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") + proto.RegisterType((*MaxShards)(nil), "internal.MaxShards") + proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") @@ -1111,10 +1112,10 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x20 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } if len(m.View) > 0 { dAtA[i] = 0x2a @@ -1212,7 +1213,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *MaxSlices) Marshal() (dAtA []byte, err error) { +func (m *MaxShards) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1222,7 +1223,7 @@ func (m *MaxSlices) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { +func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1246,7 +1247,7 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1256,7 +1257,7 @@ func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1267,10 +1268,10 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -1685,11 +1686,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { } i += n12 } - if m.MaxSlices != nil { + if m.MaxShards != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n13, err := m.MaxSlices.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxShards.Size())) + n13, err := m.MaxShards.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1980,10 +1981,10 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x28 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -2140,6 +2141,24 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2212,8 +2231,8 @@ func (m *BlockDataRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPrivate(uint64(m.Block)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } l = len(m.View) if l > 0 { @@ -2255,7 +2274,7 @@ func (m *Cache) Size() (n int) { return n } -func (m *MaxSlices) Size() (n int) { +func (m *MaxShards) Size() (n int) { var l int _ = l if len(m.Standard) > 0 { @@ -2269,15 +2288,15 @@ func (m *MaxSlices) Size() (n int) { return n } -func (m *CreateSliceMessage) Size() (n int) { +func (m *CreateShardMessage) Size() (n int) { var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -2454,8 +2473,8 @@ func (m *NodeStatus) Size() (n int) { l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.MaxSlices != nil { - l = m.MaxSlices.Size() + if m.MaxShards != nil { + l = m.MaxShards.Size() n += 1 + l + sovPrivate(uint64(l)) } if m.Schema != nil { @@ -2591,8 +2610,8 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -3140,9 +3159,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3152,7 +3171,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3493,7 +3512,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } return nil } -func (m *MaxSlices) Unmarshal(dAtA []byte) error { +func (m *MaxShards) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3516,10 +3535,10 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group") + return fmt.Errorf("proto: MaxShards: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: MaxShards: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3548,14 +3567,51 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 + if iNdEx < postIndex { + var valuekey uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3565,69 +3621,31 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + valuekey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy } + m.Standard[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.Standard[mapkey] = mapvalue } - m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3650,7 +3668,7 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { +func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3673,10 +3691,10 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateShardMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateShardMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3710,9 +3728,9 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3722,7 +3740,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5053,7 +5071,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxShards", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5077,10 +5095,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.MaxSlices == nil { - m.MaxSlices = &MaxSlices{} + if m.MaxShards == nil { + m.MaxShards = &MaxShards{} } - if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.MaxShards.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6080,9 +6098,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6092,7 +6110,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6681,70 +6699,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1028 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, - 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, - 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, - 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, - 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, - 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, - 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, - 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, - 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, - 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, - 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, - 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, - 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, - 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, - 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, - 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, - 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, - 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, - 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, - 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, - 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, - 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, - 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, - 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, - 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, - 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, - 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, - 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, - 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, - 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, - 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, - 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, - 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, - 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, - 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, - 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, - 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, - 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, - 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, - 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, - 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, - 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, - 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, - 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, - 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, - 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, - 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, - 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, - 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, - 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, - 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, - 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, - 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, - 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, - 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, - 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, - 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, - 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, - 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, - 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, - 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, - 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, - 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, - 0x0a, 0x0b, 0x00, 0x00, + // 1027 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0x45, + 0x18, 0x66, 0xbd, 0x6b, 0xc7, 0x7e, 0x53, 0x87, 0x64, 0x0a, 0x61, 0x8b, 0x50, 0x6a, 0x46, 0x95, + 0x1a, 0x7a, 0x88, 0x4a, 0x7b, 0xe1, 0xab, 0x52, 0x14, 0x3b, 0xc0, 0x02, 0x09, 0x30, 0x9b, 0xf4, + 0xd6, 0xc3, 0xd4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0x96, 0xdd, 0xd9, 0x24, 0xee, 0x81, 0x2b, 0x5c, + 0xb8, 0x23, 0x7e, 0x09, 0x3f, 0x81, 0x23, 0x3f, 0x01, 0x85, 0x3f, 0x82, 0xe6, 0x9d, 0xd9, 0x8f, + 0xc4, 0x4e, 0x53, 0x85, 0xde, 0xe6, 0xfd, 0x7e, 0xe6, 0xfd, 0x9a, 0x81, 0x7e, 0x9a, 0x45, 0x27, + 0x5c, 0x89, 0xad, 0x34, 0x93, 0x4a, 0x92, 0x6e, 0x94, 0x28, 0x91, 0x25, 0x3c, 0xa6, 0x77, 0xa1, + 0x17, 0x24, 0x13, 0x71, 0xb6, 0x27, 0x14, 0x27, 0x04, 0xbc, 0x6f, 0xc5, 0x2c, 0xf7, 0xdd, 0x81, + 0xb3, 0xd9, 0x65, 0x78, 0xa6, 0x7f, 0x3a, 0x70, 0xeb, 0xcb, 0x48, 0xc4, 0x93, 0xef, 0x53, 0x15, + 0xc9, 0x24, 0x27, 0x1f, 0x40, 0x6f, 0xc8, 0xc7, 0x47, 0xe2, 0x60, 0x96, 0x0a, 0xd4, 0xec, 0xb1, + 0x9a, 0x51, 0x49, 0xc3, 0xe8, 0xa5, 0xf0, 0xbd, 0x81, 0xb3, 0xd9, 0x67, 0x35, 0x83, 0x0c, 0x60, + 0xf9, 0x20, 0x9a, 0x8a, 0x1f, 0x0b, 0x9e, 0xa8, 0x62, 0xea, 0xb7, 0xd1, 0xba, 0xc9, 0xd2, 0x10, + 0xd0, 0x71, 0x17, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xee, 0x45, 0x89, 0xdf, 0x1b, 0x38, 0x9b, 0x2e, + 0xd3, 0x47, 0xe4, 0xf0, 0x33, 0x1f, 0x2c, 0x87, 0x9f, 0x55, 0xd0, 0x97, 0x1b, 0xd0, 0x29, 0xac, + 0x04, 0xd3, 0x54, 0x66, 0x8a, 0x89, 0x3c, 0x95, 0x49, 0x8e, 0x9e, 0x76, 0xb3, 0xcc, 0x77, 0xd0, + 0xb9, 0x3e, 0xd2, 0x9f, 0x61, 0x75, 0x27, 0x96, 0xe3, 0xe3, 0x11, 0x57, 0x9c, 0x89, 0x9f, 0x0a, + 0x91, 0x2b, 0xf2, 0x0e, 0xb4, 0x31, 0x27, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x79, 0xf0, 0x5b, 0x86, + 0x8b, 0x84, 0xe6, 0xa2, 0x3d, 0x66, 0xc2, 0x63, 0x86, 0xd0, 0xdc, 0xf0, 0x88, 0x67, 0x13, 0xcc, + 0x80, 0xc7, 0x0c, 0xa1, 0x31, 0x3e, 0x8d, 0xc4, 0xa9, 0xbd, 0x36, 0x9e, 0x69, 0x00, 0x6b, 0x8d, + 0xf8, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, 0x59, + 0x0a, 0x93, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x51, 0x0b, 0x45, 0x35, 0x83, 0xde, 0x81, 0x36, 0x66, + 0x5a, 0xdf, 0xb2, 0xb6, 0xd5, 0x47, 0xfa, 0x8b, 0x03, 0xbd, 0x3d, 0x7e, 0x86, 0x30, 0x72, 0xf2, + 0x04, 0xba, 0xa1, 0xe2, 0xc9, 0x44, 0x03, 0xd4, 0x4a, 0xcb, 0x8f, 0x3e, 0xdc, 0x2a, 0x1b, 0x62, + 0xab, 0x52, 0xdb, 0x2a, 0x75, 0x76, 0x13, 0x95, 0xcd, 0x58, 0x65, 0xf2, 0xfe, 0xe7, 0xd0, 0xbf, + 0x20, 0xd2, 0xf1, 0x8e, 0xc5, 0xac, 0xcc, 0xea, 0xb1, 0x98, 0xe9, 0xfb, 0x9f, 0xf0, 0xb8, 0x10, + 0x98, 0x2b, 0x8f, 0x19, 0xe2, 0xb3, 0xd6, 0x27, 0x0e, 0xdd, 0x06, 0x32, 0xcc, 0x04, 0x57, 0x02, + 0x83, 0xec, 0x89, 0x3c, 0xe7, 0x2f, 0xc4, 0xd5, 0x19, 0x37, 0x59, 0x6c, 0x35, 0xb2, 0x48, 0x1f, + 0x00, 0x19, 0x89, 0x58, 0x28, 0x61, 0xfb, 0xf6, 0x15, 0x1e, 0x68, 0x58, 0x46, 0xbb, 0x5e, 0x97, + 0xdc, 0x07, 0x4f, 0x0f, 0x01, 0x06, 0x5b, 0x7e, 0x74, 0xbb, 0xce, 0x48, 0x35, 0x1f, 0x0c, 0x15, + 0x68, 0x5c, 0x3a, 0xc5, 0x0e, 0xb8, 0xf6, 0x0a, 0x0b, 0x9a, 0xe6, 0x81, 0x0d, 0xe5, 0x62, 0xa8, + 0xf5, 0x3a, 0x54, 0x73, 0xd0, 0x6c, 0xb4, 0xed, 0xf2, 0xba, 0x37, 0x8d, 0x46, 0x9f, 0x59, 0xae, + 0xee, 0xbf, 0x7d, 0x3e, 0x15, 0xd6, 0x06, 0xcf, 0x15, 0x94, 0xd6, 0xf5, 0x50, 0xb4, 0x7b, 0xdd, + 0xb3, 0x7a, 0x3f, 0xb8, 0xda, 0x3d, 0x12, 0xf4, 0x31, 0x74, 0xc2, 0xf1, 0x91, 0x98, 0x72, 0xf2, + 0x11, 0x2c, 0x21, 0x0e, 0x91, 0xdb, 0xb6, 0x7a, 0xfb, 0x52, 0x12, 0x59, 0x29, 0xa7, 0x23, 0x8b, + 0x7f, 0x21, 0xa6, 0xfb, 0xd0, 0xc1, 0xe8, 0xb9, 0xef, 0x5d, 0x76, 0x83, 0x7c, 0x66, 0xc5, 0x74, + 0x17, 0xdc, 0x43, 0x16, 0xe8, 0x71, 0x41, 0x04, 0xa5, 0x17, 0x4b, 0x69, 0xdf, 0x5f, 0xcb, 0x5c, + 0xd9, 0x6c, 0xe0, 0x59, 0xf3, 0x7e, 0x90, 0x99, 0xc2, 0xd4, 0xf7, 0x19, 0x9e, 0xe9, 0x33, 0xf0, + 0xf6, 0xe5, 0x44, 0x90, 0x15, 0x68, 0x05, 0x23, 0xeb, 0xa3, 0x15, 0x8c, 0xc8, 0x5d, 0x74, 0x6f, + 0x53, 0xd3, 0xaf, 0x41, 0x1c, 0xb2, 0x80, 0x61, 0xe0, 0x7b, 0xd0, 0x0f, 0xf2, 0xa1, 0x94, 0xd9, + 0x24, 0x4a, 0xb8, 0x92, 0x99, 0x5d, 0x9c, 0x17, 0x99, 0x74, 0x1b, 0x56, 0xb5, 0xfb, 0x50, 0x71, + 0x25, 0xca, 0xfa, 0xad, 0x43, 0x47, 0xf3, 0xaa, 0x70, 0x96, 0xc2, 0x96, 0xd7, 0x7a, 0x65, 0x05, + 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0xbb, 0x27, 0x22, 0x51, 0x8d, 0x0e, 0x40, 0x1a, 0x1d, 0xf4, 0x99, + 0x21, 0x08, 0x35, 0x57, 0xb1, 0x98, 0x57, 0x6a, 0xcc, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, + 0x4a, 0x40, 0x45, 0x5e, 0x99, 0x38, 0x57, 0x9b, 0x90, 0x8f, 0x1b, 0xeb, 0x63, 0x7e, 0x40, 0x2a, + 0x11, 0x6b, 0x2c, 0x99, 0xcd, 0xb2, 0x2d, 0x6c, 0x97, 0xaf, 0xd6, 0xfa, 0x86, 0x6f, 0xcb, 0xc4, + 0x69, 0x04, 0xfd, 0x61, 0x5c, 0xe4, 0x4a, 0x64, 0x16, 0x91, 0x5e, 0x73, 0x86, 0x51, 0xe5, 0xa7, + 0x66, 0x2c, 0x4e, 0x11, 0xb9, 0x07, 0x6d, 0x8d, 0xd4, 0xf4, 0xe6, 0xfc, 0x35, 0x8c, 0x90, 0x3e, + 0x85, 0xee, 0x4e, 0x18, 0x7c, 0x95, 0xc9, 0x22, 0x5d, 0xd8, 0x79, 0xe5, 0xeb, 0xd3, 0x9a, 0x7f, + 0x7d, 0xdc, 0xb9, 0xd7, 0xc7, 0xab, 0x5e, 0x1f, 0x1a, 0xc2, 0x9a, 0x59, 0x09, 0x7a, 0x24, 0x6e, + 0xb2, 0x11, 0xca, 0xa7, 0xc1, 0x6d, 0x3c, 0x0d, 0x21, 0xac, 0x99, 0xc9, 0x7f, 0x93, 0x4e, 0xff, + 0x68, 0xc1, 0x1a, 0x13, 0x79, 0xf4, 0x52, 0x04, 0x49, 0xae, 0xb2, 0x62, 0xac, 0x07, 0x5c, 0xdb, + 0x7f, 0x23, 0x9f, 0xdb, 0x6c, 0xbb, 0xcc, 0x10, 0xaf, 0xd3, 0x4c, 0xe4, 0x21, 0x2c, 0x5f, 0x1e, + 0x80, 0x79, 0xd5, 0xa6, 0x0a, 0x79, 0x08, 0x4b, 0xa1, 0x2c, 0xb2, 0xb1, 0x28, 0xc7, 0xbb, 0xb1, + 0x74, 0x0c, 0x32, 0x23, 0x66, 0xa5, 0x5a, 0xa3, 0x95, 0xda, 0xaf, 0x6e, 0x25, 0xf2, 0xe4, 0x52, + 0x2b, 0xf9, 0x1d, 0x34, 0x78, 0xaf, 0x36, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0xaf, 0x0e, 0xdc, + 0x6a, 0x42, 0x78, 0xad, 0xd9, 0xa8, 0x2a, 0xd2, 0x5a, 0x58, 0x11, 0x77, 0x51, 0x45, 0xbc, 0xba, + 0x22, 0xf5, 0x2b, 0xd7, 0x6e, 0xbe, 0x72, 0xc7, 0x70, 0x67, 0xae, 0x4c, 0x43, 0x39, 0x4d, 0x75, + 0x3f, 0xfc, 0x8f, 0x72, 0xe9, 0xad, 0x91, 0x65, 0xb6, 0x50, 0x3d, 0x66, 0x08, 0xfa, 0x29, 0xbc, + 0x1b, 0x0a, 0xd5, 0x28, 0x52, 0xd9, 0x6d, 0x03, 0x70, 0xf7, 0xc5, 0xe9, 0x15, 0xd7, 0xd7, 0x22, + 0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0x23, 0xeb, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6, + 0xf2, 0xc5, 0xec, 0x9a, 0xa9, 0xf7, 0x61, 0xc9, 0xac, 0x48, 0xf3, 0xf1, 0xe9, 0xb1, 0x92, 0xa4, + 0xb7, 0x75, 0x43, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, 0x43, 0xff, 0x80, 0xf2, 0x9d, 0xd5, 0xbf, + 0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0x7f, 0xce, 0x37, 0x9c, 0xdf, 0xff, 0xdd, 0x78, 0xeb, + 0x79, 0x07, 0x7f, 0xbe, 0x8f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x39, 0x2f, 0x93, 0x68, 0x0a, + 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 23bb4886c..408c065e5 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -24,7 +24,7 @@ message BlockDataRequest { string Index = 1; string Field = 2; string View = 5; - uint64 Slice = 4; + uint64 Shard = 4; uint64 Block = 3; } @@ -37,13 +37,13 @@ message Cache { repeated uint64 IDs = 1; } -message MaxSlices { +message MaxShards { map Standard = 1; } -message CreateSliceMessage { +message CreateShardMessage { string Index = 1; - uint64 Slice = 2; + uint64 Shard = 2; } message DeleteIndexMessage { @@ -105,7 +105,7 @@ message NodeEventMessage { message NodeStatus { Node Node = 1; - MaxSlices MaxSlices = 2; + MaxShards MaxShards = 2; Schema Schema = 3; } @@ -148,7 +148,7 @@ message ResizeSource { string Index = 2; string Field = 3; string View = 4; - uint64 Slice = 5; + uint64 Shard = 5; } message ResizeInstructionComplete { diff --git a/internal/public.pb.go b/internal/public.pb.go index 3cb6fa270..cd1927f06 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: public.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" - import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -268,7 +267,7 @@ func (m *AttrMap) GetAttrs() []*Attr { type QueryRequest struct { Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` @@ -287,9 +286,9 @@ func (m *QueryRequest) GetQuery() string { return "" } -func (m *QueryRequest) GetSlices() []uint64 { +func (m *QueryRequest) GetShards() []uint64 { if m != nil { - return m.Slices + return m.Shards } return nil } @@ -413,7 +412,7 @@ func (m *QueryResult) GetChanged() bool { type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` @@ -440,9 +439,9 @@ func (m *ImportRequest) GetField() string { return "" } -func (m *ImportRequest) GetSlice() uint64 { +func (m *ImportRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -485,7 +484,7 @@ func (m *ImportRequest) GetTimestamps() []int64 { type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` @@ -510,9 +509,9 @@ func (m *ImportValueRequest) GetField() string { return "" } -func (m *ImportValueRequest) GetSlice() uint64 { +func (m *ImportValueRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -800,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i += 8 + i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) } return i, nil } @@ -857,10 +855,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) i += copy(dAtA[i:], m.Query) } - if len(m.Slices) > 0 { - dAtA4 := make([]byte, len(m.Slices)*10) + if len(m.Shards) > 0 { + dAtA4 := make([]byte, len(m.Shards)*10) var j3 int - for _, num := range m.Slices { + for _, num := range m.Shards { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1062,10 +1060,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { dAtA8 := make([]byte, len(m.RowIDs)*10) @@ -1179,10 +1177,10 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { dAtA14 := make([]byte, len(m.ColumnIDs)*10) @@ -1237,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1378,9 +1394,9 @@ func (m *QueryRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if len(m.Slices) > 0 { + if len(m.Shards) > 0 { l = 0 - for _, e := range m.Slices { + for _, e := range m.Shards { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1462,8 +1478,8 @@ func (m *ImportRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) } if len(m.RowIDs) > 0 { l = 0 @@ -1512,8 +1528,8 @@ func (m *ImportValueRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { l = 0 @@ -2317,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + v = uint64(dAtA[iNdEx-8]) + v |= uint64(dAtA[iNdEx-7]) << 8 + v |= uint64(dAtA[iNdEx-6]) << 16 + v |= uint64(dAtA[iNdEx-5]) << 24 + v |= uint64(dAtA[iNdEx-4]) << 32 + v |= uint64(dAtA[iNdEx-3]) << 40 + v |= uint64(dAtA[iNdEx-2]) << 48 + v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex @@ -2497,7 +2520,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -2537,10 +2560,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shards", wireType) } case 3: if wireType != 0 { @@ -3078,9 +3101,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3090,7 +3113,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3449,9 +3472,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3461,7 +3484,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3748,49 +3771,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 699 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, - 0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e, - 0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54, - 0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d, - 0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73, - 0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26, - 0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0, - 0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe, - 0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46, - 0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee, - 0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3, - 0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5, - 0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12, - 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b, - 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98, - 0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c, - 0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf, - 0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6, - 0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda, - 0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa, - 0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3, - 0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57, - 0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2, - 0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd, - 0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10, - 0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3, - 0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95, - 0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1, - 0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59, - 0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5, - 0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab, - 0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb, - 0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94, - 0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82, - 0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9, - 0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9, - 0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7, - 0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1, - 0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f, - 0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d, - 0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66, - 0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90, - 0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00, - 0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00, + // 701 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84, + 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b, + 0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0, + 0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6, + 0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5, + 0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8, + 0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41, + 0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69, + 0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b, + 0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93, + 0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1, + 0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3, + 0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c, + 0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7, + 0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22, + 0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb, + 0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49, + 0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75, + 0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88, + 0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c, + 0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d, + 0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97, + 0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd, + 0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44, + 0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58, + 0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd, + 0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1, + 0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85, + 0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2, + 0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6, + 0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b, + 0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85, + 0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84, + 0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64, + 0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66, + 0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, + 0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9, + 0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17, + 0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09, + 0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e, + 0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8, + 0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc, + 0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index cd34b88ff..04c98d070 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -46,7 +46,7 @@ message AttrMap { message QueryRequest { string Query = 1; - repeated uint64 Slices = 2; + repeated uint64 Shards = 2; bool ColumnAttrs = 3; bool Remote = 5; bool ExcludeRowAttrs = 6; @@ -71,7 +71,7 @@ message QueryResult { message ImportRequest { string Index = 1; string Field = 2; - uint64 Slice = 3; + uint64 Shard = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; repeated string RowKeys = 7; @@ -82,7 +82,7 @@ message ImportRequest { message ImportValueRequest { string Index = 1; string Field = 2; - uint64 Slice = 3; + uint64 Shard = 3; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; diff --git a/iterator.go b/iterator.go index 473e862dd..d7526b779 100644 --- a/iterator.go +++ b/iterator.go @@ -184,11 +184,11 @@ func NewRoaringIterator(itr *roaring.Iterator) *RoaringIterator { // Seek moves the cursor to a pair matching bseek/pseek. // If the pair is not found then it moves to the next pair. func (itr *RoaringIterator) Seek(bseek, pseek uint64) { - itr.itr.Seek((bseek * SliceWidth) + pseek) + itr.itr.Seek((bseek * ShardWidth) + pseek) } // Next returns the next column/row ID pair. func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) { v, eof := itr.itr.Next() - return v / SliceWidth, v % SliceWidth, eof + return v / ShardWidth, v % ShardWidth, eof } diff --git a/pilosa.go b/pilosa.go index bff167a7d..9505bf513 100644 --- a/pilosa.go +++ b/pilosa.go @@ -56,7 +56,7 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c1291d1f7..d29f1226f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -471,10 +471,10 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.ShardWidth+5, pilosa.ShardWidth+7) + bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.ShardWidth+1) result := bm0.Difference(bm1) - if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { + if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.ShardWidth + 5, pilosa.ShardWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) } } @@ -1161,7 +1161,7 @@ func BenchmarkContainerLinear(b *testing.B) { bm := roaring.NewFileBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1172,7 +1172,7 @@ func BenchmarkContainerReverse(b *testing.B) { bm := roaring.NewFileBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1183,7 +1183,7 @@ func BenchmarkContainerColumn(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1196,8 +1196,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) { for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1209,8 +1209,8 @@ func BenchmarkContainerInsideOut(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { - bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1219,7 +1219,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(0); col < pilosa.SliceWidth; col++ { + for col := uint64(0); col < pilosa.ShardWidth; col++ { bm.Add(col) } } @@ -1228,7 +1228,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { + for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { bm.Add(col) } } diff --git a/row.go b/row.go index a59bd73e0..cbfa6b270 100644 --- a/row.go +++ b/row.go @@ -157,12 +157,12 @@ func (r *Row) Difference(other *Row) *Row { // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { - return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i) + return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } // ClearBit clears the i-th column of the row. func (r *Row) ClearBit(i uint64) (changed bool) { - s := r.segment(i / SliceWidth) + s := r.segment(i / ShardWidth) if s == nil { return false } @@ -174,24 +174,24 @@ func (r *Row) Segments() []RowSegment { return r.segments } -// segment returns a segment for a given slice. +// segment returns a segment for a given shard. // Returns nil if segment does not exist. -func (r *Row) segment(slice uint64) *RowSegment { +func (r *Row) segment(shard uint64) *RowSegment { if i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice - }); i < len(r.segments) && r.segments[i].slice == slice { + return r.segments[i].shard >= shard + }); i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } return nil } -func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment { +func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment { i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice + return r.segments[i].shard >= shard }) // Return exact match. - if i < len(r.segments) && r.segments[i].slice == slice { + if i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } @@ -202,7 +202,7 @@ func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment { } r.segments[i] = RowSegment{ data: *roaring.NewBitmap(), - slice: slice, + shard: shard, writable: true, } @@ -218,7 +218,7 @@ func (r *Row) InvalidateCount() { // IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased. func (r *Row) IncrementCount(i uint64) { - seg := r.segment(i / SliceWidth) + seg := r.segment(i / ShardWidth) if seg != nil { seg.n++ } @@ -227,7 +227,7 @@ func (r *Row) IncrementCount(i uint64) { // DecrementCount decrements the row cached counter. func (r *Row) DecrementCount(i uint64) { - seg := r.segment(i / SliceWidth) + seg := r.segment(i / ShardWidth) if seg != nil { if seg.n > 0 { seg.n-- @@ -308,10 +308,10 @@ func Union(rows []*Row) *Row { // RowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The -// width of the segment will always match the slice width. +// width of the segment will always match the shard width. type RowSegment struct { - // Slice this segment belongs to - slice uint64 + // Shard this segment belongs to + shard uint64 // Underlying raw bitmap implementation. // This is an mmapped bitmap if writable is false. Otherwise @@ -345,7 +345,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -356,7 +356,7 @@ func (s *RowSegment) Union(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -367,7 +367,7 @@ func (s *RowSegment) Difference(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -378,7 +378,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -464,15 +464,15 @@ func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) { } // Otherwise determine which is first. - if s0.slice < s1.slice { + if s0.shard < s1.shard { itr.a0 = itr.a0[1:] return s0, nil - } else if s0.slice > s1.slice { + } else if s0.shard > s1.shard { itr.a1 = itr.a1[1:] return s1, nil } - // Return both if slices are equal. + // Return both if shards are equal. itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:] return s0, s1 } diff --git a/row_test.go b/row_test.go index bc1cc0c68..7f1279ceb 100644 --- a/row_test.go +++ b/row_test.go @@ -30,7 +30,7 @@ func TestRow_Merge(t *testing.T) { exp uint64 }{ { - r1: pilosa.NewRow(1, 2, 3, SliceWidth+1, 2*SliceWidth), + r1: pilosa.NewRow(1, 2, 3, ShardWidth+1, 2*ShardWidth), r2: pilosa.NewRow(3, 4, 5), exp: 7, }, @@ -56,9 +56,9 @@ func TestRow_Merge(t *testing.T) { // Ensure a row can Xor'ed func TestRow_Xor(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth, 2 * ShardWidth} res := r1.Xor(r2) if res.Count() != 3 { @@ -78,9 +78,9 @@ func TestRow_Xor(t *testing.T) { } func TestRow_Union_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{0, 1, ShardWidth, 2 * ShardWidth} res := r1.Union(r2) if res.Count() != 4 { @@ -99,9 +99,9 @@ func TestRow_Union_Segment(t *testing.T) { } func TestRow_Difference_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth} res := r1.Difference(r2) if res.Count() != 2 { diff --git a/server.go b/server.go index c07720a54..b04661f53 100644 --- a/server.go +++ b/server.go @@ -412,12 +412,12 @@ func (s *Server) monitorAntiEntropy() { // ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { - case *internal.CreateSliceMessage: + case *internal.CreateShardMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - idx.SetRemoteMaxSlice(obj.Slice) + idx.SetRemoteMaxShard(obj.Shard) case *internal.CreateIndexMessage: opt := IndexOptions{} _, err := s.holder.CreateIndex(obj.Index, opt) @@ -537,7 +537,7 @@ func (s *Server) Node() *Node { // where a node fails to receive a Broadcast message, or // when a new (empty) node needs to get in sync with the // rest of the cluster, two things are shared via gossip: -// - MaxSlice by Index +// - MaxShard by Index // - Schema // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalStatus() (proto.Message, error) { @@ -550,7 +550,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { ns := internal.NodeStatus{ Node: EncodeNode(s.cluster.Node), - MaxSlices: s.holder.EncodeMaxSlices(), + MaxShards: s.holder.EncodeMaxShards(), Schema: s.holder.EncodeSchema(), } @@ -593,19 +593,19 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return errors.Wrap(err, "applying schema") } - // Sync maxSlices. - oldmaxslices := s.holder.MaxSlices() - for index, newMax := range ns.MaxSlices.Standard { + // Sync maxShards. + oldmaxshards := s.holder.MaxShards() + for index, newMax := range ns.MaxShards.Standard { localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation + // indexes should be created and synced prior to shard creation if localIndex == nil { s.logger.Printf("Local Index not found: %s", index) continue } - if newMax > oldmaxslices[index] { - oldmaxslices[index] = newMax - localIndex.SetRemoteMaxSlice(newMax) + if newMax > oldmaxshards[index] { + oldmaxshards[index] = newMax + localIndex.SetRemoteMaxShard(newMax) } } diff --git a/server/cluster_test.go b/server/cluster_test.go index d8c2d7802..aff676ae0 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -95,22 +95,22 @@ func TestMain_SendReceiveMessage(t *testing.T) { // We have to wait for the broadcast message to be sent before checking state. time.Sleep(1 * time.Second) - // Make sure node0 knows about the latest MaxSlice. - maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + // Make sure node0 knows about the latest MaxShard. + maxShards0, err := client0.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices0["i"] != 2 { - t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + if maxShards0["i"] != 2 { + t.Fatalf("unexpected maxShard on node0: %d", maxShards0["i"]) } - // Make sure node1 knows about the latest MaxSlice. - maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + // Make sure node1 knows about the latest MaxShard. + maxShards1, err := client1.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices1["i"] != 2 { - t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + if maxShards1["i"] != 2 { + t.Fatalf("unexpected maxShard on node1: %d", maxShards1["i"]) } } @@ -181,7 +181,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("ContinuousSlices", func(t *testing.T) { + t.Run("ContinuousShards", func(t *testing.T) { // Configure node0 m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() @@ -222,7 +222,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("SkippedSlice", func(t *testing.T) { + t.Run("SkippedShard", func(t *testing.T) { // Configure node0 m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() @@ -239,7 +239,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } - // Write data on first node. Note that no data is placed on slice 1. + // Write data on first node. Note that no data is placed on shard 1. if _, err := m0.Query("i", "", ` Set(1, f=1) Set(2400000, f=1) @@ -390,7 +390,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.ShardWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index 2981f375c..169065971 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -55,7 +55,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { + } else if body := w.Body.String(); body != fmt.Sprintf("{\"shardWidth\":%d}\n", pilosa.ShardWidth) { t.Fatalf("unexpected body: %s", body) } }) @@ -112,19 +112,19 @@ func TestHandler_Endpoints(t *testing.T) { // TODO need to test aborting a cluster resize job. this may not be the right place }) - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) - hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.ShardWidth)+4) hldr.SetBit("i0", "f0", 31, 1) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+8) - t.Run("Max Slice", func(t *testing.T) { + t.Run("Max Shard", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/shards/max", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { @@ -132,9 +132,9 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Slices args", func(t *testing.T) { + t.Run("Shards args", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { @@ -142,11 +142,11 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Slices args protobuf", func(t *testing.T) { + t.Run("Shards args protobuf", func(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ Query: "Count(Row(f0=30))", - Slices: []uint64{0, 1}, + Shards: []uint64{0, 1}, }) if err != nil { t.Fatal(err) @@ -169,17 +169,17 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query args error", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=a,b", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"invalid shard argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) } }) t.Run("Query params err", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { @@ -217,9 +217,9 @@ func TestHandler_Endpoints(t *testing.T) { }) f0 := i0.Field("f0") - if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil { + if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) } else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil { t.Fatal(err) @@ -249,7 +249,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -285,7 +285,7 @@ func TestHandler_Endpoints(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) @@ -301,7 +301,7 @@ func TestHandler_Endpoints(t *testing.T) { if a := resp.ColumnAttrSets; len(a) != 2 { t.Fatalf("unexpected column attributes length: %d", len(a)) - } else if a[0].ID != pilosa.SliceWidth+1 { + } else if a[0].ID != pilosa.ShardWidth+1 { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected column attr length: %d", len(a)) @@ -376,7 +376,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Err Parse", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?shards=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { @@ -507,7 +507,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Fragment Nodes", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil) + r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -520,7 +520,7 @@ func TestHandler_Endpoints(t *testing.T) { // invalid argument should return BadRequest w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) @@ -528,7 +528,7 @@ func TestHandler_Endpoints(t *testing.T) { // index is required w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) diff --git a/stats_test.go b/stats_test.go index 932672e3a..3452f2b8c 100644 --- a/stats_test.go +++ b/stats_test.go @@ -39,43 +39,43 @@ func TestMultiStatClient_Expvar(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, SliceWidth) - hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) hldr.ClearBit("d", "f", 0, 1) - if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Gauge creates a unique key, subsequent Gauge calls will overwrite hldr.Stats.Gauge("g", 5, 1.0) hldr.Stats.Gauge("g", 8, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Set creates a unique key, subsequent sets will overwrite hldr.Stats.Set("s", "4", 1.0) hldr.Stats.Set("s", "7", 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Record timing duration and a uniquely Set key/value dur, _ := time.ParseDuration("123us") hldr.Stats.Timing("tt", dur, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Expvar histogram is implemented as a gauge hldr.Stats.Histogram("hh", 3, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } @@ -91,8 +91,8 @@ func TestStatsCount_TopN(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, SliceWidth) - hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) // Execute query. called := false diff --git a/test/fragment.go b/test/fragment.go index 6faf5d72d..90d2fa291 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -18,8 +18,8 @@ import ( "github.com/pilosa/pilosa" ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth +// ShardWidth is a helper reference to use when testing. +const ShardWidth = pilosa.ShardWidth // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { diff --git a/test/handler.go b/test/handler.go index 048d9b883..944cfb8ee 100644 --- a/test/handler.go +++ b/test/handler.go @@ -64,13 +64,13 @@ func MustNewHandler(opts ...http.HandlerOption) *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) + ExecuteFn func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, index, query, slices, opt) +func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return c.ExecuteFn(ctx, index, query, shards, opt) } // Server represents a test wrapper for httptest.Server. diff --git a/test/holder.go b/test/holder.go index 648d910ed..3d09c9e34 100644 --- a/test/holder.go +++ b/test/holder.go @@ -90,7 +90,7 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { } // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { +func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, shard uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { @@ -100,7 +100,7 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, if err != nil { panic(err) } - frag, err := v.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { panic(err) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 5e19a83b9..7843ed443 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -119,9 +119,9 @@ func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) erro func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / SliceWidth - nodes := c0.sliceNodes(index, slice) + c0 := t.Clusters[0] // use the first node's cluster to determine shard location. + shard := colID / ShardWidth + nodes := c0.shardNodes(index, shard) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -355,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) + // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) instrNode := DecodeNode(instr.Node) destCluster := t.clusterByID(instrNode.ID) @@ -368,14 +368,14 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi srcNode := DecodeNode(src.Node) srcCluster := t.clusterByID(srcNode.ID) - srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice) - destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice) + srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard) + destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard) if destFragment == nil { // Create fragment on destination if it doesn't exist. f := destCluster.holder.Field(src.Index, src.Field) v := f.View(src.View) var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + destFragment, err = v.CreateFragmentIfNotExists(src.Shard) if err != nil { return err } diff --git a/view.go b/view.go index 732ecd485..25b90cfb4 100644 --- a/view.go +++ b/view.go @@ -49,13 +49,13 @@ type View struct { cacheSize uint32 - // Fragments by slice. + // Fragments by shard. cacheType string // passed in by field fragments map[uint64]*Fragment - // maxSlice maintains this view's max slice in order to - // prevent sending multiple `CreateSliceMessage` messages - maxSlice uint64 + // maxShard maintains this view's max shard in order to + // prevent sending multiple `CreateShardMessage` messages + maxShard uint64 broadcaster Broadcaster stats StatsClient @@ -132,17 +132,17 @@ func (v *View) openFragments() error { } // Parse filename into integer. - slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { continue } - frag := v.newFragment(v.fragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) + return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.slice] = frag + v.fragments[frag.shard] = frag } return nil @@ -164,15 +164,15 @@ func (v *View) close() error { return nil } -// calculateMaxSlice returns the max slice in the view. -func (v *View) calculateMaxSlice() uint64 { +// calculateMaxShard returns the max shard in the view. +func (v *View) calculateMaxShard() uint64 { v.mu.RLock() defer v.mu.RUnlock() var max uint64 - for slice := range v.fragments { - if slice > max { - max = slice + for shard := range v.fragments { + if shard > max { + max = shard } } @@ -180,18 +180,18 @@ func (v *View) calculateMaxSlice() uint64 { } // fragmentPath returns the path to a fragment in the view. -func (v *View) fragmentPath(slice uint64) string { - return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) +func (v *View) fragmentPath(shard uint64) string { + return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10)) } -// Fragment returns a fragment in the view by slice. -func (v *View) Fragment(slice uint64) *Fragment { +// Fragment returns a fragment in the view by shard. +func (v *View) Fragment(shard uint64) *Fragment { v.mu.RLock() defer v.mu.RUnlock() - return v.fragment(slice) + return v.fragment(shard) } -func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } +func (v *View) fragment(shard uint64) *Fragment { return v.fragments[shard] } // allFragments returns a list of all fragments in the view. func (v *View) allFragments() []*Fragment { @@ -212,64 +212,64 @@ func (v *View) recalculateCaches() { } } -// CreateFragmentIfNotExists returns a fragment in the view by slice. -func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { +// CreateFragmentIfNotExists returns a fragment in the view by shard. +func (v *View) CreateFragmentIfNotExists(shard uint64) (*Fragment, error) { v.mu.Lock() defer v.mu.Unlock() - return v.createFragmentIfNotExists(slice) + return v.createFragmentIfNotExists(shard) } -func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { +func (v *View) createFragmentIfNotExists(shard uint64) (*Fragment, error) { // Find fragment in cache first. - if frag := v.fragments[slice]; frag != nil { + if frag := v.fragments[shard]; frag != nil { return frag, nil } // Initialize and open fragment. - frag := v.newFragment(v.fragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } frag.RowAttrStore = v.RowAttrStore - // Broadcast a message that a new max slice was just created. - if slice > v.maxSlice { - v.maxSlice = slice + // Broadcast a message that a new max shard was just created. + if shard > v.maxShard { + v.maxShard = shard - // Send the create slice message to all nodes. + // Send the create shard message to all nodes. err := v.broadcaster.SendSync( - &internal.CreateSliceMessage{ + &internal.CreateShardMessage{ Index: v.index, - Slice: slice, + Shard: shard, }) if err != nil { - return nil, errors.Wrap(err, "sending createslice message") + return nil, errors.Wrap(err, "sending createshard message") } } // Save to lookup. - v.fragments[slice] = frag + v.fragments[shard] = frag return frag, nil } -func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.index, v.field, v.name, slice) +func (v *View) newFragment(path string, shard uint64) *Fragment { + frag := NewFragment(path, v.index, v.field, v.name, shard) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.Logger - frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) + frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) return frag } // deleteFragment removes the fragment from the view. -func (v *View) deleteFragment(slice uint64) error { +func (v *View) deleteFragment(shard uint64) error { - fragment := v.fragments[slice] + fragment := v.fragments[shard] if fragment == nil { return ErrFragmentNotFound } - v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -283,15 +283,15 @@ func (v *View) deleteFragment(slice uint64) error { // Delete fragment cache file. if err := os.Remove(fragment.cachePath()); err != nil { - v.Logger.Printf("no cache file to delete for slice %d", slice) + v.Logger.Printf("no cache file to delete for shard %d", shard) } - delete(v.fragments, slice) + delete(v.fragments, shard) return nil } -// row returns a row for a slice of the view. +// row returns a row for a shard of the view. func (v *View) row(rowID uint64) *Row { row := NewRow() for _, frag := range v.allFragments() { @@ -307,8 +307,8 @@ func (v *View) row(rowID uint64) *Row { // setBit sets a bit within the view. func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } @@ -317,8 +317,8 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { // clearBit clears a bit within the view. func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } @@ -327,8 +327,8 @@ func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { // value uses a column of bits to read a multi-bit value. func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return value, exists, err } @@ -337,8 +337,8 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, // setValue uses a column of bits to set a multi-bit value. func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } diff --git a/view_internal_test.go b/view_internal_test.go index d0e8bfdd1..a592330cb 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -39,27 +39,27 @@ func TestView_DeleteFragment(t *testing.T) { v := mustOpenView("i", "f", "v") defer v.close() - slice := uint64(9) + shard := uint64(9) // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) + fragment, err := v.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } else if fragment == nil { t.Fatal("expected fragment") } - err = v.deleteFragment(slice) + err = v.deleteFragment(shard) if err != nil { t.Fatal(err) } - if v.Fragment(slice) != nil { + if v.Fragment(shard) != nil { t.Fatal("fragment still exists in view") } - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) + // Recreate fragment with same shard, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } else if fragment == fragment2 {