scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into more-gossip-stuff

Browse source
This commit is contained in:
Matt Jaffee 2018-06-29 06:49:13 -05:00
commit 2e15db7ce0
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
53 changed files with 1398 additions and 1268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
NOTES
View file

@ -14,13 +14,13 @@
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
F ▶│0000000000000000000000000000000000000000│
r ││0000000000000000000000000000000000000000│
a ││0000000000000000000000000000000000000000│
m ││0000000000000000000000000000000000000000│
e ▶│0000000000000000000000000000000000000000│
i ││0000000000000000000000000000000000000000│
e ││0000000000000000000000000000000000000000│
l ││0000000000000000000000000000000000000000│
d ▶│0000000000000000000000000000000000000000│
└────────────────────────────────────────┘
▲───────────▲
Slice
Shard
Fragment=intersection of frame & slice
Fragment=intersection of field & shard

110
api.go
View file

@ -35,10 +35,9 @@ import (
// API provides the top level programmatic interface to Pilosa. It is usually
// wrapped by a handler which provides an external interface (e.g. HTTP).
type API struct {
Holder *Holder
Broadcaster Broadcaster
Cluster *Cluster
server *Server
Holder *Holder
Cluster *Cluster
server *Server
}
// APIOption is a functional option type for pilosa.API
@ -48,7 +47,6 @@ func OptAPIServer(s *Server) APIOption {
return func(a *API) error {
a.server = s
a.Holder = s.holder
a.Broadcaster = s
a.Cluster = s.cluster
return nil
}
@ -56,11 +54,7 @@ func OptAPIServer(s *Server) APIOption {
// NewAPI returns a new API instance.
func NewAPI(opts ...APIOption) (*API, error) {
api := &API{
Broadcaster: NopBroadcaster,
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
}
api := &API{}
for _, opt := range opts {
err := opt(api)
@ -115,7 +109,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
ExcludeRowAttrs: req.ExcludeRowAttrs,
ExcludeColumns: req.ExcludeColumns,
}
results, err := api.server.executor.Execute(ctx, req.Index, q, req.Slices, execOpts)
results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {
return resp, errors.Wrap(err, "executing")
}
@ -190,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
return nil, errors.Wrap(err, "creating index")
}
// Send the create index message to all nodes.
err = api.Broadcaster.SendSync(
err = api.server.SendSync(
&internal.CreateIndexMessage{
Index: indexName,
Meta: options.Encode(),
@ -229,7 +223,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
return errors.Wrap(err, "deleting index")
}
// Send the delete index message to all nodes.
err = api.Broadcaster.SendSync(
err = api.server.SendSync(
&internal.DeleteIndexMessage{
Index: indexName,
})
@ -269,7 +263,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
}
// Send the create field message to all nodes.
err = api.Broadcaster.SendSync(
err = api.server.SendSync(
&internal.CreateFieldMessage{
Index: indexName,
Field: fieldName,
@ -303,7 +297,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
}
// Send the delete field message to all nodes.
err := api.Broadcaster.SendSync(
err := api.server.SendSync(
&internal.DeleteFieldMessage{
Index: indexName,
Field: fieldName,
@ -316,21 +310,21 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
return nil
}
// ExportCSV encodes the fragment designated by the index,field,slice as
// ExportCSV encodes the fragment designated by the index,field,shard as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error {
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error {
if err := api.validate(apiExportCSV); err != nil {
return errors.Wrap(err, "validating api method")
}
// Validate that this handler owns the slice.
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return ErrClusterDoesNotOwnSlice
// Validate that this handler owns the shard.
if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
return ErrClusterDoesNotOwnShard
}
// Find the fragment.
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard)
if f == nil {
return ErrFragmentNotFound
}
@ -354,25 +348,25 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
return nil
}
// SliceNodes returns the node and all replicas which should contain a slice's data.
func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) {
if err := api.validate(apiSliceNodes); err != nil {
// ShardNodes returns the node and all replicas which should contain a shard's data.
func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) {
if err := api.validate(apiShardNodes); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
return api.Cluster.sliceNodes(indexName, slice), nil
return api.Cluster.shardNodes(indexName, shard), nil
}
// MarshalFragment returns an object which can write the specified fragment's data
// to an io.Writer. The serialized data can be read back into a fragment with
// the UnmarshalFragment API call.
func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) {
func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) {
if err := api.validate(apiMarshalFragment); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -382,7 +376,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName
// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
// Reader which was previously written by MarshalFragment to populate the
// fragment's data.
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error {
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error {
if err := api.validate(apiUnmarshalFragment); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -400,7 +394,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa
}
// Retrieve fragment from field.
frag, err := view.CreateFragmentIfNotExists(slice)
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
@ -430,7 +424,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice)
f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Shard)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -448,13 +442,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) {
func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, shard uint64) ([]FragmentBlock, error) {
if err := api.validate(apiFragmentBlocks); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -476,7 +470,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error {
return errors.Wrap(err, "validating api method")
}
err := api.Broadcaster.SendSync(&internal.RecalculateCaches{})
err := api.server.SendSync(&internal.RecalculateCaches{})
if err != nil {
return errors.Wrap(err, "broacasting message")
}
@ -552,14 +546,14 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
// Delete the view.
if err := f.DeleteView(viewName); err != nil {
// Ignore this error because views do not exist on all nodes due to slice distribution.
// Ignore this error because views do not exist on all nodes due to shard distribution.
if err != ErrInvalidView {
return errors.Wrap(err, "deleting view")
}
}
// Send the delete view message to all nodes.
err := api.Broadcaster.SendSync(
err := api.server.SendSync(
&internal.DeleteViewMessage{
Index: indexName,
Field: fieldName,
@ -641,13 +635,13 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s
return attrs, nil
}
// Import bulk imports data into a particular index,field,slice.
// Import bulk imports data into a particular index,field,shard.
func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting field")
}
@ -665,7 +659,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
// Import into fragment.
err = field.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -676,21 +670,21 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
return errors.Wrap(err, "validating api method")
}
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = field.ImportValue(req.ColumnIDs, req.Values)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
// MaxSlices returns the maximum slice number for each index in a map.
func (api *API) MaxSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxSlices()
// MaxShards returns the maximum shard number for each index in a map.
func (api *API) MaxShards(ctx context.Context) map[string]uint64 {
return api.Holder.MaxShards()
}
// StatsWithTags returns an instance of whatever implementation of StatsClient
@ -711,25 +705,25 @@ func (api *API) LongQueryTime() time.Duration {
return api.Cluster.longQueryTime
}
func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) {
// Validate that this handler owns the slice.
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return nil, nil, ErrClusterDoesNotOwnSlice
func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) {
// Validate that this handler owns the shard.
if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName)
return nil, nil, ErrClusterDoesNotOwnShard
}
// Find the Index.
api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, slice)
api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard)
index := api.Holder.Index(indexName)
if index == nil {
api.server.logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error())
api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error())
return nil, nil, ErrIndexNotFound
}
// Retrieve field.
field := index.Field(fieldName)
if field == nil {
api.server.logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error())
api.server.logger.Printf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error())
return nil, nil, ErrFieldNotFound
}
return index, field, nil
@ -753,7 +747,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
}
// Send the set-coordinator message to new node.
err = api.Broadcaster.SendTo(
err = api.server.SendTo(
newNode,
&internal.SetCoordinatorMessage{
New: EncodeNode(newNode),
@ -851,12 +845,12 @@ func (api *API) Version() string {
// Info returns information about this server instance
func (api *API) Info() ServerInfo {
return ServerInfo{
SliceWidth: SliceWidth,
ShardWidth: ShardWidth,
}
}
type ServerInfo struct {
SliceWidth uint64 `json:"sliceWidth"`
ShardWidth uint64 `json:"shardWidth"`
}
type apiMethod int
@ -881,14 +875,14 @@ const (
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
//apiMaxSlices // not implemented
//apiMaxShards // not implemented
apiQuery
apiRecalculateCaches
apiRemoveNode
apiResizeAbort
//apiSchema // not implemented
apiSetCoordinator
apiSliceNodes
apiShardNodes
//apiState // not implemented
//apiStatsWithTags // not implemented
apiUnmarshalFragment
@ -923,7 +917,7 @@ var methodsNormal = map[apiMethod]struct{}{
apiQuery: struct{}{},
apiRecalculateCaches: struct{}{},
apiRemoveNode: struct{}{},
apiSliceNodes: struct{}{},
apiShardNodes: struct{}{},
apiUnmarshalFragment: struct{}{},
apiViews: struct{}{},
}

6
apimethod_string.go
View file

@ -2,15 +2,15 @@
package pilosa
import "fmt"
import "strconv"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return fmt.Sprintf("apiMethod(%d)", i)
return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

10
broadcast.go
View file

@ -56,7 +56,7 @@ func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error {
// Broadcast message types.
const (
messageTypeCreateSlice = iota
messageTypeCreateShard = iota
messageTypeCreateIndex
messageTypeDeleteIndex
messageTypeCreateField
@ -77,8 +77,8 @@ const (
func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateSliceMessage:
typ = messageTypeCreateSlice
case *internal.CreateShardMessage:
typ = messageTypeCreateShard
case *internal.CreateIndexMessage:
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
@ -123,8 +123,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
var m proto.Message
switch typ {
case messageTypeCreateSlice:
m = &internal.CreateSliceMessage{}
case messageTypeCreateShard:
m = &internal.CreateShardMessage{}
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
case messageTypeDeleteIndex:

4
broadcast_test.go
View file

@ -26,9 +26,9 @@ import (
// Ensure a message can be marshaled and unmarshaled.
func TestMessage_Marshal(t *testing.T) {
testMessageMarshal(t, &internal.CreateSliceMessage{
testMessageMarshal(t, &internal.CreateShardMessage{
Index: "i",
Slice: 8,
Shard: 8,
})
testMessageMarshal(t, &internal.DeleteIndexMessage{

32
client.go
View file

@ -32,25 +32,25 @@ type FieldValue struct {
// While I understand that putting the entire Client behind an interface might require this many methods,
// I don't want to let it go unquestioned.
type InternalClient interface {
MaxSliceByIndex(ctx context.Context) (map[string]uint64, error)
MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error
Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureField(ctx context.Context, indexName string, fieldName string) error
ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error
ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error)
FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error)
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
}
//===============
@ -81,7 +81,7 @@ func NewNopInternalClient() *NopInternalClient {
var _ InternalClient = NewNopInternalClient()
func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
func (n *NopInternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
return nil, nil
}
func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
@ -90,7 +90,7 @@ func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) {
func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
return nil, nil
}
func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
@ -99,7 +99,7 @@ func (n *NopInternalClient) Query(ctx context.Context, index string, queryReques
func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) {
return nil, nil
}
func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error {
func (n *NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error {
return nil
}
func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error {
@ -111,19 +111,19 @@ func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, option
func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error {
func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error {
return nil
}
func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error {
func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
return nil
}
func (n *NopInternalClient) CreateField(ctx context.Context, index, field string) error {
return nil
}
func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) {
func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) {
func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
@ -135,6 +135,6 @@ func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fi
func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error {
return nil
}
func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) {
func (n *NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {
return nil, nil
}

64
cluster.go
View file

@ -576,7 +576,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool {
type frag struct {
field string
view string
slice uint64
shard uint64
}
func fragsDiff(a, b []frag) []frag {
@ -623,15 +623,15 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
}
}
return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews)
return c.fragCombos(idx.Name(), idx.MaxShard(), fieldViews)
}
// fragCombos returns a map (by uri) of lists of fragments for a given index
// by creating every combination of field/view specified in `fieldViews` up to maxSlice.
func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost {
// by creating every combination of field/view specified in `fieldViews` up to maxShard.
func (c *Cluster) fragCombos(idx string, maxShard uint64, fieldViews viewsByField) fragsByHost {
t := make(fragsByHost)
for i := uint64(0); i <= maxSlice; i++ {
nodes := c.sliceNodes(idx, i)
for i := uint64(0); i <= maxShard; i++ {
nodes := c.shardNodes(idx, i)
for _, n := range nodes {
// for each field/view combination:
for field, views := range fieldViews {
@ -762,7 +762,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
Index: idx.Name(),
Field: frag.field,
View: frag.view,
Slice: frag.slice,
Shard: frag.shard,
}
m[nodeID] = append(m[nodeID], src)
@ -772,10 +772,10 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
return m, nil
}
// partition returns the partition that a slice belongs to.
func (c *Cluster) partition(index string, slice uint64) int {
// partition returns the partition that a shard belongs to.
func (c *Cluster) partition(index string, shard uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
binary.BigEndian.PutUint64(buf[:], shard)
// Hash the bytes and mod by partition count.
h := fnv.New64a()
@ -784,14 +784,14 @@ func (c *Cluster) partition(index string, slice uint64) int {
return int(h.Sum64() % uint64(c.partitionN))
}
// sliceNodes returns a list of nodes that own a fragment.
func (c *Cluster) sliceNodes(index string, slice uint64) []*Node {
return c.partitionNodes(c.partition(index, slice))
// shardNodes returns a list of nodes that own a fragment.
func (c *Cluster) shardNodes(index string, shard uint64) []*Node {
return c.partitionNodes(c.partition(index, shard))
}
// ownsSlice returns true if a host owns a fragment.
func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool {
return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID)
// ownsShard returns true if a host owns a fragment.
func (c *Cluster) ownsShard(nodeID string, index string, shard uint64) bool {
return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID)
}
// partitionNodes returns a list of nodes that own a partition.
@ -817,20 +817,20 @@ func (c *Cluster) partitionNodes(partitionID int) []*Node {
return nodes
}
// containsSlices is like OwnsSlices, but it includes replicas.
func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
// containsShards is like OwnsShards, but it includes replicas.
func (c *Cluster) containsShards(index string, maxShard uint64, node *Node) []uint64 {
var shards []uint64
for i := uint64(0); i <= maxShard; i++ {
p := c.partition(index, i)
// Determine the nodes for partition.
nodes := c.partitionNodes(p)
for _, n := range nodes {
if n.ID == node.ID {
slices = append(slices, i)
shards = append(shards, i)
}
}
}
return slices
return shards
}
// Hasher represents an interface to hash integers into buckets.
@ -1211,7 +1211,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
// Request each source file in ResizeSources.
for _, src := range instr.Sources {
c.logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
srcURI := decodeURI(src.Node.URI)
@ -1228,27 +1228,27 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
}
// Create the local fragment.
frag, err := v.CreateFragmentIfNotExists(src.Slice)
frag, err := v.CreateFragmentIfNotExists(src.Shard)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
// Stream slice from remote node.
c.logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI)
// Stream shard from remote node.
c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
// on the remote node. This occurs when a slice has been skipped and
// on the remote node. This occurs when a shard has been skipped and
// therefore doesn't contain data. The coordinator correctly determined
// the resize instruction to retrieve the slice, but it doesn't have data.
// the resize instruction to retrieve the shard, but it doesn't have data.
// TODO: figure out a way to distinguish from "fragment not found" errors
// which are true errors and which simply mean the fragment doesn't have data.
if err == ErrFragmentNotFound {
return nil
}
return errors.Wrap(err, "retrieving slice")
return errors.Wrap(err, "retrieving shard")
} else if rd == nil {
return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI)
return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI)
}
// Write to local field and always close reader.
@ -1257,7 +1257,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err
_, err := frag.ReadFrom(rd)
return err
}(); err != nil {
return errors.Wrap(err, "copying remote slice")
return errors.Wrap(err, "copying remote shard")
}
}
return nil

24
cluster_internal_test.go
View file

@ -49,13 +49,13 @@ func TestFragCombos(t *testing.T) {
tests := []struct {
idx string
maxSlice uint64
maxShard uint64
fieldViews viewsByField
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
maxShard: uint64(2),
fieldViews: viewsByField{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
@ -64,7 +64,7 @@ func TestFragCombos(t *testing.T) {
},
{
idx: "foo",
maxSlice: uint64(3),
maxShard: uint64(3),
fieldViews: viewsByField{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
@ -74,7 +74,7 @@ func TestFragCombos(t *testing.T) {
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews)
actual := c.fragCombos(test.idx, test.maxShard, test.fieldViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
@ -339,13 +339,13 @@ func TestCluster_Owners(t *testing.T) {
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
if err := quick.Check(func(index string, shard uint64, partitionN int) bool {
c := NewCluster()
c.partitionN = partitionN
partitionID := c.partition(index, slice)
partitionID := c.partition(index, shard)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
}
return true
@ -380,14 +380,14 @@ func TestHasher(t *testing.T) {
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
// Ensure ContainsShards can find the actual shard list for node and index.
func TestCluster_ContainsShards(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.containsSlices("test", 10, c.Nodes[2])
shards := c.containsShards("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected shars for node's index: %v", shards)
}
}

2
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and field. The data
of the CSV file are grouped by slice for the most efficient import.
of the CSV file are grouped by shard for the most efficient import.
The format of the CSV file is:

14
ctl/export.go
View file

@ -80,16 +80,16 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
return errors.Wrap(err, "creating client")
}
// Determine slice count.
maxSlices, err := client.MaxSliceByIndex(ctx)
// Determine shard count.
maxShards, err := client.MaxShardByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting slice count")
return errors.Wrap(err, "getting shard count")
}
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil {
// Export each shard.
for shard := uint64(0); shard <= maxShards[cmd.Index]; shard++ {
logger.Printf("exporting shard: %d", shard)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, shard, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

2
ctl/export_test.go
View file

@ -44,7 +44,7 @@ func TestExportCommand_Validation(t *testing.T) {
}
func TestExportCommand_Run(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)

22
ctl/import.go
View file

@ -118,7 +118,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
}
}
// Import each path and import by slice.
// Import each path and import by shard.
for _, path := range cmd.Paths {
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, fieldType, path); err != nil {
@ -243,18 +243,18 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Group bits by slice.
// Group bits by shard.
logger.Printf("grouping %d bits", len(bits))
bitsBySlice := http.Bits(bits).GroupBySlice()
bitsByShard := http.Bits(bits).GroupByShard()
// Parse path into bits.
for slice, chunk := range bitsBySlice {
for shard, chunk := range bitsByShard {
if cmd.Sort {
sort.Sort(http.BitsByPos(chunk))
}
logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil {
logger.Printf("importing shard: %d, n=%d", shard, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}
@ -437,18 +437,18 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error {
func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Group vals by slice.
// Group vals by shard.
logger.Printf("grouping %d vals", len(vals))
valsBySlice := http.FieldValues(vals).GroupBySlice()
valsByShard := http.FieldValues(vals).GroupByShard()
// Parse path into FieldValues.
for slice, vals := range valsBySlice {
for shard, vals := range valsByShard {
if cmd.Sort {
sort.Sort(http.FieldValues(vals))
}
logger.Printf("importing slice: %d, n=%d", slice, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil {
logger.Printf("importing shard: %d, n=%d", shard, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil {
return errors.Wrap(err, "importing values")
}
}

8
ctl/import_test.go
View file

@ -61,7 +61,7 @@ func TestImportCommand_Run(t *testing.T) {
t.Fatal(err)
}
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
@ -86,7 +86,7 @@ func TestImportCommand_RunValue(t *testing.T) {
t.Fatal(err)
}
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@ -102,7 +102,7 @@ func TestImportCommand_RunValue(t *testing.T) {
}
func TestImportCommand_InvalidFile(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
@ -176,7 +176,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
}
func TestImportCommand_BugOverwriteValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)

6
diagnostics.go
View file

@ -216,14 +216,14 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
var numShards uint64
numFields := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range d.server.holder.Indexes() {
numSlices += index.MaxSlice() + 1
numShards += index.MaxShard() + 1
numIndexes += 1
for _, field := range index.Fields() {
numFields += 1
@ -238,7 +238,7 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
d.Set("NumIndexes", numIndexes)
d.Set("NumFields", numFields)
d.Set("NumSlices", numSlices)
d.Set("NumShards", numShards)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}

300
executor.go
View file

@ -37,7 +37,7 @@ const (
rowLabel = "row"
)
// Executor recursively executes calls in a PQL query across all slices.
// Executor recursively executes calls in a PQL query across all shards.
type Executor struct {
Holder *Holder
@ -80,7 +80,7 @@ func NewExecutor(opts ...ExecutorOption) *Executor {
}
// Execute executes a PQL query.
func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) {
// Verify that an index is set.
if index == "" {
return nil, ErrIndexRequired
@ -108,7 +108,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
}
}
results, err := e.execute(ctx, index, q, slices, opt)
results, err := e.execute(ctx, index, q, shards, opt)
if err != nil {
return nil, err
}
@ -123,24 +123,24 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
return results, nil
}
func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Don't bother calculating slices for query types that don't require it.
needsSlices := needsSlices(q.Calls)
func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) {
// Don't bother calculating shards for query types that don't require it.
needsShards := needsShards(q.Calls)
// If slices are specified, then use that value for slices. If slices aren't
// If shards are specified, then use that value for shards. If shards aren't
// specified, then include all of them.
if len(slices) == 0 && needsSlices {
// Round up the number of slices.
if len(shards) == 0 && needsShards {
// Round up the number of shards.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxShard := idx.MaxShard()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
// Generate a slice of all shards.
shards = make([]uint64, maxShard+1)
for i := range shards {
shards[i] = uint64(i)
}
}
@ -152,7 +152,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
v, err := e.executeCall(ctx, index, call, slices, opt)
v, err := e.executeCall(ctx, index, call, shards, opt)
if err != nil {
return nil, err
}
@ -162,7 +162,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic
}
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) {
if err := e.validateCallArgs(c); err != nil {
return nil, errors.Wrap(err, "validating args")
}
@ -171,18 +171,18 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
switch c.Name {
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
return e.executeSum(ctx, index, c, shards, opt)
case "Min":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeMin(ctx, index, c, slices, opt)
return e.executeMin(ctx, index, c, shards, opt)
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeMax(ctx, index, c, slices, opt)
return e.executeMax(ctx, index, c, shards, opt)
case "Clear":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, slices, opt)
return e.executeCount(ctx, index, c, shards, opt)
case "Set":
return e.executeSetBit(ctx, index, c, opt)
case "SetValue":
@ -193,10 +193,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return nil, e.executeSetColumnAttrs(ctx, index, c, opt)
case "TopN":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeTopN(ctx, index, c, slices, opt)
return e.executeTopN(ctx, index, c, shards, opt)
default:
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeBitmapCall(ctx, index, c, slices, opt)
return e.executeBitmapCall(ctx, index, c, shards, opt)
}
}
@ -220,7 +220,7 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
}
// executeSum executes a Sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Sum(): field required")
}
@ -230,8 +230,8 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeSumCountSlice(ctx, index, c, slice)
mapFn := func(shard uint64) (interface{}, error) {
return e.executeSumCountShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -240,7 +240,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
return other.Add(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
@ -253,7 +253,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
}
// executeMin executes a Min() call.
func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
@ -263,8 +263,8 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeMinSlice(ctx, index, c, slice)
mapFn := func(shard uint64) (interface{}, error) {
return e.executeMinShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -273,7 +273,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl
return other.Smaller(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
@ -286,7 +286,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl
}
// executeMax executes a Max() call.
func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
@ -296,8 +296,8 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeMaxSlice(ctx, index, c, slice)
mapFn := func(shard uint64) (interface{}, error) {
return e.executeMaxShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -306,7 +306,7 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl
return other.Larger(v.(ValCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return ValCount{}, err
}
@ -319,10 +319,10 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Row, error) {
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeBitmapCallSlice(ctx, index, c, slice)
mapFn := func(shard uint64) (interface{}, error) {
return e.executeBitmapCallShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -335,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
return other
}
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
@ -384,31 +384,31 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
return row, nil
}
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeBitmapCallShard executes a bitmap call for a single shard.
func (e *Executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
switch c.Name {
case "Row":
return e.executeBitmapSlice(ctx, index, c, slice)
return e.executeBitmapShard(ctx, index, c, shard)
case "Difference":
return e.executeDifferenceSlice(ctx, index, c, slice)
return e.executeDifferenceShard(ctx, index, c, shard)
case "Intersect":
return e.executeIntersectSlice(ctx, index, c, slice)
return e.executeIntersectShard(ctx, index, c, shard)
case "Range":
return e.executeRangeSlice(ctx, index, c, slice)
return e.executeRangeShard(ctx, index, c, shard)
case "Union":
return e.executeUnionSlice(ctx, index, c, slice)
return e.executeUnionShard(ctx, index, c, shard)
case "Xor":
return e.executeXorSlice(ctx, index, c, slice)
return e.executeXorShard(ctx, index, c, shard)
default:
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
}
// executeSumCountSlice calculates the sum and count for bsiGroups on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
// executeSumCountShard calculates the sum and count for bsiGroups on a shard.
func (e *Executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return ValCount{}, errors.Wrap(err, "executing bitmap call")
}
@ -427,7 +427,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return ValCount{}, nil
}
@ -442,11 +442,11 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
}, nil
}
// executeMinSlice calculates the min for bsiGroups on a slice.
func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
// executeMinShard calculates the min for bsiGroups on a shard.
func (e *Executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return ValCount{}, err
}
@ -465,7 +465,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return ValCount{}, nil
}
@ -480,11 +480,11 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal
}, nil
}
// executeMaxSlice calculates the max for bsiGroups on a slice.
func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
// executeMaxShard calculates the max for bsiGroups on a shard.
func (e *Executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return ValCount{}, err
}
@ -503,7 +503,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return ValCount{}, nil
}
@ -521,7 +521,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) {
idsArg, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
@ -532,7 +532,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
}
// Execute original query.
pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt)
pairs, err := e.executeTopNShards(ctx, index, c, shards, opt)
if err != nil {
return nil, errors.Wrap(err, "finding top results")
}
@ -549,7 +549,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
sort.Sort(uint64Slice(ids))
other.Args["ids"] = ids
trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt)
trimmedList, err := e.executeTopNShards(ctx, index, other, shards, opt)
if err != nil {
return nil, errors.Wrap(err, "retrieving full counts")
}
@ -560,10 +560,10 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s
return trimmedList, nil
}
func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeTopNSlice(ctx, index, c, slice)
mapFn := func(shard uint64) (interface{}, error) {
return e.executeTopNShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -572,7 +572,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
return Pairs(other).Add(v.([]Pair))
}
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
@ -584,32 +584,32 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
return results, nil
}
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
// executeTopNShard executes a TopN call for a single shard.
func (e *Executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) {
field, _ := c.Args["_field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
return nil, fmt.Errorf("executeTopNShard: %v", err)
}
attrName, _ := c.Args["attrName"].(string)
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
return nil, fmt.Errorf("executeTopNShard: %v", err)
}
minThreshold, _, err := c.UintArg("threshold")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
return nil, fmt.Errorf("executeTopNShard: %v", err)
}
attrValues, _ := c.Args["attrValues"].([]interface{})
tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
return nil, fmt.Errorf("executeTopNShard: %v", err)
}
// Retrieve bitmap used to intersect.
var src *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
@ -623,7 +623,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
field = defaultField
}
f := e.Holder.Fragment(index, field, ViewStandard, slice)
f := e.Holder.Fragment(index, field, ViewStandard, shard)
if f == nil {
return nil, nil
}
@ -646,14 +646,14 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
})
}
// executeDifferenceSlice executes a difference() call for a local slice.
func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeDifferenceShard executes a difference() call for a local shard.
func (e *Executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Difference query is currently not supported")
}
for i, input := range c.Children {
row, err := e.executeBitmapCallSlice(ctx, index, input, slice)
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
@ -668,7 +668,7 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *
return other, nil
}
func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
func (e *Executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
// Fetch column label from index.
idx := e.Holder.Index(index)
if idx == nil {
@ -693,21 +693,21 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice)
frag := e.Holder.Fragment(index, fieldName, ViewStandard, shard)
if frag == nil {
return NewRow(), nil
}
return frag.row(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeIntersectShard executes a intersect() call for a local shard.
func (e *Executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Intersect query is currently not supported")
}
for i, input := range c.Children {
row, err := e.executeBitmapCallSlice(ctx, index, input, slice)
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
@ -722,11 +722,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p
return other, nil
}
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeRangeShard executes a range() call for a local shard.
func (e *Executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
return e.executeBSIGroupRangeSlice(ctx, index, c, slice)
return e.executeBSIGroupRangeShard(ctx, index, c, shard)
}
// Parse field.
@ -750,7 +750,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Read row & column id.
rowID, rowOK, err := c.UintArg(fieldName)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
return nil, fmt.Errorf("executeRangeShard - reading row: %v", err)
}
if !rowOK {
return nil, fmt.Errorf("Range() must specify %q", rowLabel)
@ -785,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, fieldName, view, slice)
f := e.Holder.Fragment(index, fieldName, view, shard)
if f == nil {
continue
}
@ -795,8 +795,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return row, nil
}
// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice.
func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard.
func (e *Executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
// Only one conditional should be present.
if len(c.Args) == 0 {
return nil, errors.New("Range(): condition required")
@ -836,7 +836,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if frag == nil {
return NewRow(), nil
}
@ -857,7 +857,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
// The reason we don't just call:
// return f.RangeBetween(fieldName, predicates[0], predicates[1])
// here is because we need the call to be slice-specific.
// here is because we need the call to be shard-specific.
// Find bsiGroup.
bsig := f.bsiGroup(fieldName)
@ -871,7 +871,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if frag == nil {
return NewRow(), nil
}
@ -904,7 +904,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if frag == nil {
return NewRow(), nil
}
@ -925,11 +925,11 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string,
}
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeUnionShard executes a union() call for a local shard.
func (e *Executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
other := NewRow()
for i, input := range c.Children {
row, err := e.executeBitmapCallSlice(ctx, index, input, slice)
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
@ -944,11 +944,11 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C
return other, nil
}
// executeXorSlice executes a xor() call for a local slice.
func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// executeXorShard executes a xor() call for a local shard.
func (e *Executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
other := NewRow()
for i, input := range c.Children {
row, err := e.executeBitmapCallSlice(ctx, index, input, slice)
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
@ -964,7 +964,7 @@ func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Cal
}
// executeCount executes a count() call.
func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (uint64, error) {
if len(c.Children) == 0 {
return 0, errors.New("Count() requires an input bitmap")
} else if len(c.Children) > 1 {
@ -972,8 +972,8 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
mapFn := func(shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return 0, err
}
@ -986,7 +986,7 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
@ -1032,12 +1032,12 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
// executeClearBitField executes a Clear() call for a single view.
func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.sliceNodes(index, slice) {
for _, node := range e.Cluster.shardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.ClearBit(rowID, colID, nil)
val, err := f.ClearBit(rowID, colID)
if err != nil {
return false, err
} else if val {
@ -1107,10 +1107,10 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
// executeSetBitField executes a Set() call for a specific view.
func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.sliceNodes(index, slice) {
for _, node := range e.Cluster.shardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
val, err := f.SetBit(rowID, colID, timestamp)
@ -1389,12 +1389,12 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
return nil
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
// exec executes a PQL query remotely for a set of shards on a node.
func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *ExecOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
Query: q.String(),
Slices: slices,
Shards: shards,
Remote: true,
}
@ -1439,29 +1439,29 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
return results, nil
}
// slicesByNode returns a mapping of nodes to slices.
// Returns errSliceUnavailable if a slice cannot be allocated to a node.
func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) {
// shardsByNode returns a mapping of nodes to shards.
// Returns errShardUnavailable if a shard cannot be allocated to a node.
func (e *Executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) {
m := make(map[*Node][]uint64)
loop:
for _, slice := range slices {
for _, node := range e.Cluster.sliceNodes(index, slice) {
for _, shard := range shards {
for _, node := range e.Cluster.shardNodes(index, shard) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
m[node] = append(m[node], shard)
continue loop
}
}
return nil, errSliceUnavailable
return nil, errShardUnavailable
}
return m, nil
}
// mapReduce maps and reduces data across the cluster.
//
// If a mapping of slices to a node fails then the slices are resplit across
// If a mapping of shards to a node fails then the shards are resplit across
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
func (e *Executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
ch := make(chan mapResponse)
// Wrap context with a cancel to kill goroutines on exit.
@ -1480,13 +1480,13 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
}
// Start mapping across all primary owners.
if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil {
if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil {
return nil, errors.Wrap(err, "starting mapper")
}
// Iterate over all map responses and reduce.
var result interface{}
var maxSlice int
var maxShard int
for {
select {
case <-ctx.Done():
@ -1500,7 +1500,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable {
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, err
@ -1511,32 +1511,32 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
// Reduce value.
result = reduceFn(result, resp.result)
// If all slices have been processed then return.
maxSlice += len(resp.slices)
if maxSlice >= len(slices) {
// If all shards have been processed then return.
maxShard += len(resp.shards)
if maxShard >= len(shards) {
return result, nil
}
}
}
}
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
// Group slices together by nodes.
m, err := e.slicesByNode(nodes, index, slices)
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return err
}
// Execute each node in a separate goroutine.
for n, nodeSlices := range m {
go func(n *Node, nodeSlices []uint64) {
resp := mapResponse{node: n, slices: nodeSlices}
for n, nodeShards := range m {
go func(n *Node, nodeShards []uint64) {
resp := mapResponse{node: n, shards: nodeShards}
// Send local slices to mapper, otherwise remote exec.
// Send local shards to mapper, otherwise remote exec.
if n.ID == e.Node.ID {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, opt)
if len(results) > 0 {
resp.result = results[0]
}
@ -1548,30 +1548,30 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
case <-ctx.Done():
case ch <- resp:
}
}(n, nodeSlices)
}(n, nodeShards)
}
return nil
}
// mapperLocal performs map & reduce entirely on the local node.
func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
ch := make(chan mapResponse, len(slices))
func (e *Executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
ch := make(chan mapResponse, len(shards))
for _, slice := range slices {
go func(slice uint64) {
result, err := mapFn(slice)
for _, shard := range shards {
go func(shard uint64) {
result, err := mapFn(shard)
// Return response to the channel.
select {
case <-ctx.Done():
case ch <- mapResponse{result: result, err: err}:
}
}(slice)
}(shard)
}
// Reduce results
var maxSlice int
var maxShard int
var result interface{}
for {
select {
@ -1582,11 +1582,11 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu
return nil, resp.err
}
result = reduceFn(result, resp.result)
maxSlice++
maxShard++
}
// Exit once all slices are processed.
if maxSlice == len(slices) {
// Exit once all shards are processed.
if maxShard == len(shards) {
return result, nil
}
}
@ -1695,16 +1695,16 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res
return result, nil
}
// errSliceUnavailable is a marker error if no nodes are available.
var errSliceUnavailable = errors.New("slice unavailable")
// errShardUnavailable is a marker error if no nodes are available.
var errShardUnavailable = errors.New("shard unavailable")
type mapFunc func(slice uint64) (interface{}, error)
type mapFunc func(shard uint64) (interface{}, error)
type reduceFunc func(prev, v interface{}) interface{}
type mapResponse struct {
node *Node
slices []uint64
shards []uint64
result interface{}
err error
@ -1740,7 +1740,7 @@ func hasOnlySetRowAttrs(calls []*pql.Call) bool {
return true
}
func needsSlices(calls []*pql.Call) bool {
func needsShards(calls []*pql.Call) bool {
if len(calls) == 0 {
return false
}

220
executor_test.go
View file

@ -46,8 +46,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -57,7 +57,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
@ -75,7 +75,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
@ -95,12 +95,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
})
@ -170,17 +170,17 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i", "general", 10, 1)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.SetBit("i", "general", 10, ShardWidth+1)
hldr.SetBit("i", "general", 10, ShardWidth+2)
hldr.SetBit("i", "general", 11, 1)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
hldr.SetBit("i", "general", 11, ShardWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -201,16 +201,16 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i", "general", 10, 0)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.SetBit("i", "general", 10, ShardWidth+1)
hldr.SetBit("i", "general", 10, ShardWidth+2)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
hldr.SetBit("i", "general", 11, ShardWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -234,16 +234,16 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i", "general", 10, 0)
hldr.SetBit("i", "general", 10, SliceWidth+1)
hldr.SetBit("i", "general", 10, SliceWidth+2)
hldr.SetBit("i", "general", 10, ShardWidth+1)
hldr.SetBit("i", "general", 10, ShardWidth+2)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, SliceWidth+2)
hldr.SetBit("i", "general", 11, ShardWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -253,8 +253,8 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i", "f", 10, 3)
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, SliceWidth+2)
hldr.SetBit("i", "f", 10, ShardWidth+1)
hldr.SetBit("i", "f", 10, ShardWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
@ -267,7 +267,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
// Ensure a set query can be executed.
func TestExecutor_Execute_SetBit(t *testing.T) {
t.Run("ID", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 1, 0)
@ -312,7 +312,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
})
t.Run("Keys", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// Set columns for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two shards.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
@ -516,12 +516,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
Set(0, f=0)
Set(1, f=0)
Set(`+strconv.Itoa(SliceWidth)+`, f=0)
Set(`+strconv.Itoa(SliceWidth+2)+`, f=0)
Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0)
Set(`+strconv.Itoa(ShardWidth)+`, f=0)
Set(`+strconv.Itoa(ShardWidth+2)+`, f=0)
Set(`+strconv.Itoa((5*ShardWidth)+100)+`, f=0)
Set(0, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=20)
Set(`+strconv.Itoa(ShardWidth)+`, f=10)
Set(`+strconv.Itoa(ShardWidth)+`, f=20)
Set(0, other=0)
`), nil, nil); err != nil {
t.Fatal(err)
@ -546,7 +546,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set columns for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two shards.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
@ -586,13 +586,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set columns for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two shards.
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 0, 2)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 1, SliceWidth+2)
hldr.SetBit("i", "f", 1, SliceWidth)
hldr.SetBit("i", "f", 0, ShardWidth)
hldr.SetBit("i", "f", 1, ShardWidth+2)
hldr.SetBit("i", "f", 1, ShardWidth)
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -611,22 +611,22 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 0, 2*SliceWidth)
hldr.SetBit("i", "f", 0, 3*SliceWidth)
hldr.SetBit("i", "f", 0, 4*SliceWidth)
hldr.SetBit("i", "f", 0, ShardWidth)
hldr.SetBit("i", "f", 0, 2*ShardWidth)
hldr.SetBit("i", "f", 0, 3*ShardWidth)
hldr.SetBit("i", "f", 0, 4*ShardWidth)
hldr.SetBit("i", "f", 1, 0)
hldr.SetBit("i", "f", 1, 1)
hldr.SetBit("i", "f", 2, SliceWidth)
hldr.SetBit("i", "f", 2, SliceWidth+1)
hldr.SetBit("i", "f", 2, ShardWidth)
hldr.SetBit("i", "f", 2, ShardWidth+1)
hldr.SetBit("i", "f", 3, 2*SliceWidth)
hldr.SetBit("i", "f", 3, 2*SliceWidth+1)
hldr.SetBit("i", "f", 3, 2*ShardWidth)
hldr.SetBit("i", "f", 3, 2*ShardWidth+1)
hldr.SetBit("i", "f", 4, 3*SliceWidth)
hldr.SetBit("i", "f", 4, 3*SliceWidth+1)
hldr.SetBit("i", "f", 4, 3*ShardWidth)
hldr.SetBit("i", "f", 4, 3*ShardWidth+1)
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -644,20 +644,20 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set columns for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two shards.
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 0, SliceWidth)
hldr.SetBit("i", "f", 10, SliceWidth)
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 20, SliceWidth)
hldr.SetBit("i", "f", 20, SliceWidth+1)
hldr.SetBit("i", "f", 20, SliceWidth+2)
hldr.SetBit("i", "f", 0, ShardWidth)
hldr.SetBit("i", "f", 10, ShardWidth)
hldr.SetBit("i", "f", 10, ShardWidth+1)
hldr.SetBit("i", "f", 20, ShardWidth)
hldr.SetBit("i", "f", 20, ShardWidth+1)
hldr.SetBit("i", "f", 20, ShardWidth+2)
// Create an intersecting row.
hldr.SetBit("i", "other", 100, SliceWidth)
hldr.SetBit("i", "other", 100, SliceWidth+1)
hldr.SetBit("i", "other", 100, SliceWidth+2)
hldr.SetBit("i", "other", 100, ShardWidth)
hldr.SetBit("i", "other", 100, ShardWidth+1)
hldr.SetBit("i", "other", 100, ShardWidth+2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
@ -683,7 +683,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, SliceWidth)
hldr.SetBit("i", "f", 10, ShardWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
@ -706,7 +706,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
defer hldr.Close()
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, SliceWidth)
hldr.SetBit("i", "f", 10, ShardWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
@ -747,18 +747,18 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
Set(0, x=0)
Set(3, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
Set(`+strconv.Itoa(ShardWidth+1)+`, x=0)
Set(1, x=1)
Set(`+strconv.Itoa(SliceWidth+2)+`, x=2)
Set(`+strconv.Itoa(ShardWidth+2)+`, x=2)
SetValue(col=0, f=20)
SetValue(col=1, f=-5)
SetValue(col=2, f=-5)
SetValue(col=3, f=10)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60)
SetValue(col=`+strconv.Itoa(ShardWidth)+`, f=30)
SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, f=40)
SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, f=50)
SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, f=60)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -857,14 +857,14 @@ func TestExecutor_Execute_Sum(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
Set(0, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
Set(`+strconv.Itoa(ShardWidth+1)+`, x=0)
SetValue(col=0, foo=20)
SetValue(col=0, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=40)
SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=50)
SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
`), nil, nil); err != nil {
t.Fatal(err)
@ -928,6 +928,18 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Clear( 2, f=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
}
// Ensure a Range(bsiGroup) query can be executed.
@ -979,14 +991,14 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
Set(0, f=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, f=0)
Set(`+strconv.Itoa(ShardWidth+1)+`, f=0)
SetValue(col=50, foo=20)
SetValue(col=50, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=10)
SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=20)
SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
SetValue(col=0, edge=100)
SetValue(col=1, edge=-100)
@ -997,7 +1009,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -1012,7 +1024,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
@ -1027,7 +1039,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -1035,7 +1047,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -1043,7 +1055,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -1051,7 +1063,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -1129,35 +1141,35 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Row(f=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
} else if !reflect.DeepEqual(shards, []uint64{1}) {
t.Fatalf("unexpected shards: %+v", shards)
}
// Set columns in slice 0 & 2.
// Set columns in shard 0 & 2.
r := pilosa.NewRow(
(0*SliceWidth)+1,
(0*SliceWidth)+2,
(2*SliceWidth)+4,
(0*ShardWidth)+1,
(0*ShardWidth)+2,
(2*ShardWidth)+4,
)
return []interface{}{r}, nil
}
// Create local executor data.
// The local node owns slice 1.
// The local node owns shard 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, ShardWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*ShardWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -1180,16 +1192,16 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
c.Nodes[1].URI = *uri
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(10)}, nil
}
// Create local executor data. The local node owns slice 1.
// Create local executor data. The local node owns shard 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 10, (2*SliceWidth)+1)
hldr.SetBit("i", "f", 10, (2*SliceWidth)+2)
hldr.SetBit("i", "f", 10, (2*ShardWidth)+1)
hldr.SetBit("i", "f", 10, (2*ShardWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
@ -1219,7 +1231,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Set(_col=2, f=10)` {
@ -1274,7 +1286,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` {
@ -1330,15 +1342,15 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if !reflect.DeepEqual(slices, []uint64{1, 3}) {
t.Fatalf("unexpected slices: %+v", slices)
} else if !reflect.DeepEqual(shards, []uint64{1, 3}) {
t.Fatalf("unexpected shards: %+v", shards)
}
// Query should be executed twice. Once to get the top bitmaps for the
// slices and a second time to get the counts for a set of bitmaps.
// shards and a second time to get the counts for a set of bitmaps.
switch remoteExecN {
case 0:
if query.String() != `TopN(_field="f", n=3)` {
@ -1361,12 +1373,12 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
}}, nil
}
// Create local executor data on slice 2 & 4.
// Create local executor data on shard 2 & 4.
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 30, (2*SliceWidth)+1)
hldr.SetBit("i", "f", 30, (4*SliceWidth)+2)
hldr.SetBit("i", "f", 30, (2*ShardWidth)+1)
hldr.SetBit("i", "f", 30, (4*ShardWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil {
@ -1396,7 +1408,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` {
@ -1407,7 +1419,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
}
// Create local executor data.
// The local node owns slice 1.
// The local node owns shard 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -1417,7 +1429,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
}
f := hldr.Field("i", "f")
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 10, SliceWidth+1)
hldr.SetBit("i", "f", 10, ShardWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil {

109
field.go
View file

@ -21,6 +21,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
@ -38,6 +39,10 @@ const (
// Default ranked field cache
DefaultCacheSize = 50000
bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64
maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1
)
// Field types.
@ -152,15 +157,15 @@ func (f *Field) Path() string { return f.path }
// RowAttrStore returns the attribute storage.
func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the field.
func (f *Field) MaxSlice() uint64 {
// MaxShard returns the max shard in the field.
func (f *Field) MaxShard() uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
var max uint64
for _, view := range f.views {
if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max {
max = viewMaxShard
}
}
return max
@ -609,7 +614,6 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) {
if view := f.views[name]; view != nil {
return view, false, nil
}
view := f.newView(f.ViewPath(name), name)
if err := view.open(); err != nil {
@ -665,7 +669,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) {
return view.row(rowID), nil
}
// ViewRow returns a row for a view and slice.
// ViewRow returns a row for a view and shard.
// TODO: unexport this with views (it's only used in tests).
func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) {
view := f.View(viewName)
@ -715,13 +719,14 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err
}
// ClearBit clears a bit within the field.
func (f *Field) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) {
func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) {
viewName := ViewStandard
// Retrieve view. Exit if it doesn't exist.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
return changed, errors.Wrap(err, "creating view")
view, present := f.views[viewName]
if !present {
return changed, errors.Wrap(err, "clearing missing view")
}
// Clear non-time bit.
@ -730,29 +735,75 @@ func (f *Field) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err e
} else if v {
changed = v
}
// Exit early if no timestamp is specified.
if t == nil {
if len(f.views) == 1 { // assuming no time views
return changed, nil
}
// If a timestamp is specified then clear bits across all views for the quantum.
for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, errors.Wrapf(err, "creating view %s", subname)
lastViewNameSize := 0
level := 0
skipAbove := maxInt
for _, view := range f.allTimeViewsSortedByQuantum() {
if lastViewNameSize < len(view.name) {
level++
} else if lastViewNameSize > len(view.name) {
level--
}
if c, err := view.clearBit(rowID, colID); err != nil {
return changed, errors.Wrapf(err, "clearing on view %s", subname)
} else if c {
changed = true
if level < skipAbove {
if changed, err = view.clearBit(rowID, colID); err != nil {
return changed, errors.Wrapf(err, "clearing on view %s", view.name)
}
if !changed {
skipAbove = level + 1
} else {
skipAbove = maxInt
}
}
lastViewNameSize = len(view.name)
}
return changed, nil
}
func groupCompare(a, b string, offset int) (lt, eq bool) {
if len(a) > offset {
a = a[:offset]
}
if len(b) > offset {
b = b[:offset]
}
v := strings.Compare(a, b)
return v < 0, v == 0
}
func (f *Field) allTimeViewsSortedByQuantum() (me []*View) {
me = make([]*View, len(f.views), len(f.views))
prefix := ViewStandard + "_"
offset := len(ViewStandard) + 1
i := 0
for _, v := range f.views {
if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views
me[i] = v
i++
}
}
me = me[:i]
year := strings.Index(me[0].name, "_") + 4
month := year + 2
day := month + 2
sort.Slice(me, func(i, j int) (lt bool) {
var eq bool
// group by quantum from year to hour
if lt, eq = groupCompare(me[i].name, me[j].name, year); eq {
if lt, eq = groupCompare(me[i].name, me[j].name, month); eq {
if lt, eq = groupCompare(me[i].name, me[j].name, day); eq {
lt = strings.Compare(me[i].name, me[j].name) > 0
}
}
}
return
})
return
}
// Value reads a field value for a column.
func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) {
bsig := f.bsiGroup(f.name)
@ -934,7 +985,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// Attach bit to each standard view.
for _, name := range standard {
key := importKey{View: name, Slice: columnID / SliceWidth}
key := importKey{View: name, Shard: columnID / ShardWidth}
data := dataByFragment[key]
data.RowIDs = append(data.RowIDs, rowID)
data.ColumnIDs = append(data.ColumnIDs, columnID)
@ -949,7 +1000,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
return errors.Wrap(err, "creating view")
}
frag, err := view.CreateFragmentIfNotExists(key.Slice)
frag, err := view.CreateFragmentIfNotExists(key.Shard)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -983,7 +1034,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
// Attach value to each bsiGroup view.
for _, name := range []string{viewName} {
key := importKey{View: name, Slice: columnID / SliceWidth}
key := importKey{View: name, Shard: columnID / ShardWidth}
data := dataByFragment[key]
data.ColumnIDs = append(data.ColumnIDs, columnID)
data.Values = append(data.Values, value)
@ -1001,7 +1052,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error {
return errors.Wrap(err, "creating view")
}
frag, err := view.CreateFragmentIfNotExists(key.Slice)
frag, err := view.CreateFragmentIfNotExists(key.Shard)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
@ -1192,7 +1243,7 @@ func (b *bsiGroup) BitDepth() uint {
// Note that in this case (because the range uses the full BitDepth 0 to 1023),
// we can't simply return 1024.
// In order to make this work, we effectively need to change the operator to LTE.
// Executor.executeBSIGroupRangeSlice() takes this into account and returns
// Executor.executeBSIGroupRangeShard() takes this into account and returns
// `frag.FieldNotNull(bsig.BitDepth())` in such instances.
func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) {
if op == pql.GT || op == pql.GTE {

68
fragment.go
View file

@ -44,8 +44,8 @@ import (
)
const (
// SliceWidth is the number of column IDs in a slice.
SliceWidth = 1048576
// ShardWidth is the number of column IDs in a shard.
ShardWidth = 1048576
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
@ -63,7 +63,7 @@ const (
defaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a field and slice in an index.
// Fragment represents the intersection of a field and shard in an index.
type Fragment struct {
mu sync.RWMutex
@ -71,7 +71,7 @@ type Fragment struct {
index string
field string
view string
slice uint64
shard uint64
// File-backed storage
path string
@ -110,13 +110,13 @@ type Fragment struct {
}
// NewFragment returns a new instance of Fragment.
func NewFragment(path, index, field, view string, slice uint64) *Fragment {
func NewFragment(path, index, field, view string, shard uint64) *Fragment {
return &Fragment{
path: path,
index: index,
field: field,
view: view,
slice: slice,
shard: shard,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
@ -151,7 +151,7 @@ func (f *Fragment) Open() error {
// Read last bit to determine max row.
pos := f.storage.Max()
f.maxRowID = pos / SliceWidth
f.maxRowID = pos / ShardWidth
f.stats.Gauge("rows", float64(f.maxRowID), 1.0)
return nil
@ -165,7 +165,7 @@ func (f *Fragment) Open() error {
// openStorage opens the storage bitmap.
func (f *Fragment) openStorage() error {
// Create a roaring bitmap to serve as storage for the slice.
// Create a roaring bitmap to serve as storage for the shard.
if f.storage == nil {
f.storage = roaring.NewFileBitmap()
}
@ -257,7 +257,7 @@ func (f *Fragment) openCache() error {
// Read in all rows by ID.
// This will cause them to be added to the cache.
for _, id := range pb.IDs {
n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth)
f.cache.BulkAdd(id, n)
}
f.cache.Invalidate()
@ -337,7 +337,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac
// Only use a subset of the containers.
// NOTE: The start & end ranges must be divisible by
data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth)
data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
// Reference bitmap subrange in storage.
// We Clone() data because otherwise row will contains pointers to containers in storage.
@ -345,7 +345,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac
row := &Row{
segments: []RowSegment{{
data: *data.Clone(),
slice: f.slice,
shard: f.shard,
writable: false,
}},
}
@ -837,9 +837,9 @@ func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64
// pos translates the row ID and column ID into a position in the storage bitmap.
func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
// Return an error if the column ID is out of the range of the fragment's slice.
minColumnID := f.slice * SliceWidth
if columnID < minColumnID || columnID >= minColumnID+SliceWidth {
// Return an error if the column ID is out of the range of the fragment's shard.
minColumnID := f.shard * ShardWidth
if columnID < minColumnID || columnID >= minColumnID+ShardWidth {
return 0, errors.New("column out of bounds")
}
return pos(rowID, columnID), nil
@ -859,7 +859,7 @@ func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error {
}
// Invoke caller's function.
err = fn(i/SliceWidth, (f.slice*SliceWidth)+(i%SliceWidth))
err = fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth))
})
return err
}
@ -1093,16 +1093,16 @@ func (f *Fragment) Blocks() []FragmentBlock {
if eof {
return nil
}
blockID := int(v / (HashBlockSize * SliceWidth))
blockID := int(v / (HashBlockSize * ShardWidth))
for {
// Check for multiple block checksums in a row.
if n := f.readContiguousChecksums(&a, blockID); n > 0 {
itr.Seek(uint64(blockID+n) * HashBlockSize * SliceWidth)
itr.Seek(uint64(blockID+n) * HashBlockSize * ShardWidth)
v, eof = itr.Next()
if eof {
break
}
blockID = int(v / (HashBlockSize * SliceWidth))
blockID = int(v / (HashBlockSize * ShardWidth))
continue
}
@ -1113,7 +1113,7 @@ func (f *Fragment) Blocks() []FragmentBlock {
// Read all values for the block.
for ; ; v, eof = itr.Next() {
// Once we hit the next block, save the value for the next iteration.
blockID = int(v / (HashBlockSize * SliceWidth))
blockID = int(v / (HashBlockSize * ShardWidth))
if blockID != h.blockID || eof {
break
}
@ -1160,9 +1160,9 @@ func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) {
f.mu.Lock()
defer f.mu.Unlock()
f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) {
rowIDs = append(rowIDs, i/SliceWidth)
columnIDs = append(columnIDs, i%SliceWidth)
f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) {
rowIDs = append(rowIDs, i/ShardWidth)
columnIDs = append(columnIDs, i%ShardWidth)
})
return
}
@ -1190,7 +1190,7 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e
// Limit upper row/column pair.
maxRowID := uint64(id+1) * HashBlockSize
maxColumnID := uint64(SliceWidth)
maxColumnID := uint64(ShardWidth)
// Create buffered iterator for local block.
itrs := make([]*BufIterator, 1, len(data)+1)
@ -1278,14 +1278,14 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e
// Set local bits.
for i := range sets[0].columnIDs {
if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil {
if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.shard*ShardWidth)+sets[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "setting")
}
}
// Clear local bits.
for i := range clears[0].columnIDs {
if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil {
if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.shard*ShardWidth)+clears[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "clearing")
}
}
@ -1423,8 +1423,8 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) {
}
func (f *Fragment) snapshot() error {
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
start := time.Now()
defer track(start, completeMessage, f.stats, f.Logger)
@ -1736,7 +1736,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) syncFragment() error {
// Determine replica set.
nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice)
nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard)
if len(nodes) == 1 {
return nil
}
@ -1752,7 +1752,7 @@ func (s *FragmentSyncer) syncFragment() error {
}
// Retrieve remote blocks.
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice)
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.shard)
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -1817,7 +1817,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var uris []*URI
var pairSets []pairSet
for _, node := range s.Cluster.sliceNodes(f.index, f.slice) {
for _, node := range s.Cluster.shardNodes(f.index, f.shard) {
if s.Node.ID == node.ID {
continue
}
@ -1831,7 +1831,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
uris = append(uris, uri)
// Only sync the standard block.
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id)
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.shard, id)
if err != nil {
return errors.Wrap(err, "getting block")
}
@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Only sync the standard block.
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j])
count++
}
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j])
count++
}
@ -1933,5 +1933,5 @@ func byteSlicesEqual(a [][]byte) bool {
// pos returns the row position of a row/column pair.
func pos(rowID, columnID uint64) uint64 {
return (rowID * SliceWidth) + (columnID % SliceWidth)
return (rowID * ShardWidth) + (columnID % ShardWidth)
}

12
fragment_internal_test.go
View file

@ -742,7 +742,7 @@ func TestFragment_TopN_NopCache(t *testing.T) {
// Ensure the fragment cache limit works
func TestFragment_TopN_CacheSize(t *testing.T) {
slice := uint64(0)
shard := uint64(0)
cacheSize := uint32(3)
// Create Index.
@ -762,7 +762,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
}
// Create fragment.
frag, err := view.CreateFragmentIfNotExists(slice)
frag, err := view.CreateFragmentIfNotExists(shard)
if err != nil {
t.Fatal(err)
}
@ -1181,7 +1181,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
for row := 0; row < 100; row++ {
val := 1
i := 0
for col := 0; col < SliceWidth/2; col++ {
for col := 0; col < ShardWidth/2; col++ {
rows[i] = uint64(row)
cols[i] = uint64(val)
val += 2
@ -1215,7 +1215,7 @@ func BenchmarkFragment_Import(b *testing.B) {
i := 0
for row := 0; row < 100; row++ {
val := 1
for col := 0; col < SliceWidth/2; col++ {
for col := 0; col < ShardWidth/2; col++ {
rows[i] = uint64(row)
cols[i] = uint64(val)
val += 2
@ -1237,7 +1237,7 @@ func BenchmarkFragment_Import(b *testing.B) {
/////////////////////////////////////////////////////////////////////
// mustOpenFragment returns a new instance of Fragment with a temporary path.
func mustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment {
func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
@ -1248,7 +1248,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string)
cacheType = DefaultCacheType
}
f := NewFragment(file.Name(), index, field, view, slice)
f := NewFragment(file.Name(), index, field, view, shard)
f.CacheType = cacheType
f.RowAttrStore = newMemAttrStore()

6
handler.go
View file

@ -12,9 +12,9 @@ type QueryRequest struct {
// The query string to parse and execute.
Query string
// The slices to include in the query execution.
// If empty, all slices are included.
Slices []uint64
// The shards to include in the query execution.
// If empty, all shards are included.
Shards []uint64
// Return column attributes, if true.
ColumnAttrs bool

42
holder.go
View file

@ -200,11 +200,11 @@ func (h *Holder) HasData() (bool, error) {
return false, nil
}
// MaxSlices returns MaxSlice map for all indexes.
func (h *Holder) MaxSlices() map[string]uint64 {
// MaxShards returns MaxShard map for all indexes.
func (h *Holder) MaxShards() map[string]uint64 {
a := make(map[string]uint64)
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxSlice()
a[index.Name()] = index.MaxShard()
}
return a
}
@ -257,10 +257,10 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
return nil
}
// EncodeMaxSlices creates and internal representation of max slices.
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
// EncodeMaxShards creates and internal representation of max shards.
func (h *Holder) EncodeMaxShards() *internal.MaxShards {
return &internal.MaxShards{
Standard: h.MaxShards(),
}
}
@ -411,13 +411,13 @@ func (h *Holder) View(index, field, name string) *View {
return f.View(name)
}
// Fragment returns the fragment for an index, field & slice.
func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment {
// Fragment returns the fragment for an index, field & shard.
func (h *Holder) Fragment(index, field, view string, shard uint64) *Fragment {
v := h.View(index, field, view)
if v == nil {
return nil
}
return v.Fragment(slice)
return v.Fragment(shard)
}
// monitorCacheFlush periodically flushes all fragment caches sequentially.
@ -619,9 +619,9 @@ func (s *HolderSyncer) SyncHolder() error {
return nil
}
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) {
for shard := uint64(0); shard <= s.Holder.Index(di.Name).MaxShard(); shard++ {
// Ignore shards that this host doesn't own.
if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) {
continue
}
@ -631,8 +631,8 @@ func (s *HolderSyncer) SyncHolder() error {
}
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
if err := s.syncFragment(di.Name, fi.Name, vi.Name, shard); err != nil {
return fmt.Errorf("fragment sync error: index=%s, field=%s, shard=%d, err=%s", di.Name, fi.Name, shard, err)
}
}
}
@ -736,7 +736,7 @@ func (s *HolderSyncer) syncField(index, name string) error {
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error {
func (s *HolderSyncer) syncFragment(index, field, view string, shard uint64) error {
// Retrieve local field.
f := s.Holder.Field(index, field)
if f == nil {
@ -750,7 +750,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err
}
// Ensure fragment exists locally.
frag, err := v.CreateFragmentIfNotExists(slice)
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
@ -800,19 +800,19 @@ func (c *HolderCleaner) CleanHolder() error {
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node)
containedShards := c.Cluster.containsShards(index.Name(), index.MaxShard(), c.Node)
// Get the fragments registered in memory.
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.allFragments() {
fragSlice := fragment.slice
fragShard := fragment.shard
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
if uint64InSlice(fragShard, containedShards) {
continue
}
// Delete fragment.
if err := view.deleteFragment(fragSlice); err != nil {
if err := view.deleteFragment(fragShard); err != nil {
return errors.Wrap(err, "deleting fragment")
}
}

44
holder_test.go
View file

@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=opening storage: unmarshal storage") {
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") {
t.Fatalf("unexpected error: %s", err)
}
})
@ -373,12 +373,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr1 := test.MustOpenHolder()
defer hldr1.Close()
s.Handler.API.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr1.Holder
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
return e.Execute(ctx, index, query, shards, opt)
}
// Mock 2-node, fully replicated cluster.
@ -400,7 +400,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
hldr0.SetBit("i", "f0", 9, ShardWidth+5)
// Set a bit to create the fragment.
hldr0.SetBit("y", "z", 0, 0)
@ -410,13 +410,13 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr1.SetBit("i", "f", 3, 10)
hldr1.SetBit("i", "f", 120, 10)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5)
hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(3)
// Set highest shard.
hldr0.Index("i").SetRemoteMaxShard(1)
hldr0.Index("y").SetRemoteMaxShard(3)
// Set up syncer.
syncer := pilosa.HolderSyncer{
@ -444,11 +444,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -482,15 +482,15 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
hldr0.SetBit("i", "f0", 9, ShardWidth+5)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7)
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4)
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5)
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(2)
// Set highest shard.
hldr0.Index("i").SetRemoteMaxShard(1)
hldr0.Index("y").SetRemoteMaxShard(2)
// Keep replication the same and ensure we get the expected results.
cluster.ReplicaN = 2
@ -520,11 +520,11 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -562,7 +562,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}

110
http/client.go
View file

@ -73,15 +73,15 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client)
// Host returns the host the client was initialized with.
func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI }
// MaxSliceByIndex returns the number of slices on a server by index.
func (c *InternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx)
// MaxShardByIndex returns the number of shards on a server by index.
func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxShardByIndex(ctx)
}
// maxSliceByIndex returns the number of slices on a server by index.
func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
// maxShardByIndex returns the number of shards on a server by index.
func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/slices/max")
u := uriPathToURL(c.defaultURI, "/shards/max")
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
@ -99,7 +99,7 @@ func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64
}
defer resp.Body.Close()
var rsp getSlicesMaxResponse
var rsp getShardsMaxResponse
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http: status=%d", resp.StatusCode)
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
@ -184,11 +184,11 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
}
}
// FragmentNodes returns a list of nodes that own a slice.
func (c *InternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) {
// FragmentNodes returns a list of nodes that own a shard.
func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) {
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/fragment/nodes")
u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode()
u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode()
// Build request.
req, err := http.NewRequest("GET", u.String(), nil)
@ -272,23 +272,23 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s
return qresp, nil
}
// Import bulk imports bits for a single slice to a host.
func (c *InternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error {
// Import bulk imports bits for a single shard to a host.
func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit) error {
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
return pilosa.ErrFieldRequired
}
buf, err := marshalImportPayload(index, field, slice, bits)
buf, err := marshalImportPayload(index, field, shard, bits)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
// Retrieve a list of nodes that own the shard.
nodes, err := c.FragmentNodes(ctx, index, shard)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
return fmt.Errorf("shard nodes: %s", err)
}
// Import to each node.
@ -343,7 +343,7 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel
}
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) {
func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
@ -353,7 +353,7 @@ func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit)
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Field: field,
Slice: slice,
Shard: shard,
RowIDs: rowIDs,
ColumnIDs: columnIDs,
Timestamps: timestamps,
@ -423,23 +423,23 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, buf
return nil
}
// ImportValue bulk imports field values for a single slice to a host.
func (c *InternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error {
// ImportValue bulk imports field values for a single shard to a host.
func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue) error {
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
return pilosa.ErrFieldRequired
}
buf, err := marshalImportValuePayload(index, field, slice, vals)
buf, err := marshalImportValuePayload(index, field, shard, vals)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
// Retrieve a list of nodes that own the shard.
nodes, err := c.FragmentNodes(ctx, index, shard)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
return fmt.Errorf("shard nodes: %s", err)
}
// Import to each node.
@ -453,7 +453,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s
}
// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) {
func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
@ -462,7 +462,7 @@ func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.
buf, err := proto.Marshal(&internal.ImportValueRequest{
Index: index,
Field: field,
Slice: slice,
Shard: shard,
ColumnIDs: columnIDs,
Values: values,
})
@ -510,18 +510,18 @@ func (c *InternalClient) importValueNode(ctx context.Context, node *pilosa.Node,
return nil
}
// ExportCSV bulk exports data for a single slice from a host to CSV format.
func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error {
// ExportCSV bulk exports data for a single shard from a host to CSV format.
func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
return pilosa.ErrFieldRequired
}
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
// Retrieve a list of nodes that own the shard.
nodes, err := c.FragmentNodes(ctx, index, shard)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
return fmt.Errorf("shard nodes: %s", err)
}
// Attempt nodes in random order.
@ -529,7 +529,7 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli
for _, i := range rand.Perm(len(nodes)) {
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil {
if err := c.exportNodeCSV(ctx, node, index, field, shard, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
@ -541,13 +541,13 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli
}
// exportNode copies a CSV export from a node to w.
func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error {
func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error {
// Create URL.
u := nodePathToURL(node, "/export")
u.RawQuery = url.Values{
"index": {index},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
"shard": {strconv.FormatUint(shard, 10)},
}.Encode()
// Generate HTTP request.
@ -578,19 +578,19 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i
return nil
}
func (c *InternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) {
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
node := &pilosa.Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, field, slice, node)
return c.backupShardNode(ctx, index, field, shard, node)
}
func (c *InternalClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) {
func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
"shard": {strconv.FormatUint(shard, 10)},
}.Encode()
// Build request.
@ -671,7 +671,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) {
func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64) ([]pilosa.FragmentBlock, error) {
if uri == nil {
uri = c.defaultURI
}
@ -679,7 +679,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
u.RawQuery = url.Values{
"index": {index},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
"shard": {strconv.FormatUint(shard, 10)},
}.Encode()
// Build request.
@ -716,11 +716,11 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
}
// BlockData returns row/column id pairs for a block.
func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) {
func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
Index: index,
Field: field,
Slice: slice,
Shard: shard,
Block: uint64(block),
})
if err != nil {
@ -952,17 +952,17 @@ func (p Bits) Timestamps() []int64 {
return other
}
// GroupBySlice returns a map of bits by slice.
func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit {
// GroupByShard returns a map of bits by shard.
func (p Bits) GroupByShard() map[uint64][]pilosa.Bit {
m := make(map[uint64][]pilosa.Bit)
for _, bit := range p {
slice := bit.ColumnID / pilosa.SliceWidth
m[slice] = append(m[slice], bit)
shard := bit.ColumnID / pilosa.ShardWidth
m[shard] = append(m[shard], bit)
}
for slice, bits := range m {
for shard, bits := range m {
sort.Sort(Bits(bits))
m[slice] = bits
m[shard] = bits
}
return m
@ -996,17 +996,17 @@ func (p FieldValues) Values() []int64 {
return other
}
// GroupBySlice returns a map of field values by slice.
func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue {
// GroupByShard returns a map of field values by shard.
func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue {
m := make(map[uint64][]pilosa.FieldValue)
for _, val := range p {
slice := val.ColumnID / pilosa.SliceWidth
m[slice] = append(m[slice], val)
shard := val.ColumnID / pilosa.ShardWidth
m[shard] = append(m[shard], val)
}
for slice, vals := range m {
for shard, vals := range m {
sort.Sort(FieldValues(vals))
m[slice] = vals
m[shard] = vals
}
return m
@ -1027,7 +1027,7 @@ func (p BitsByPos) Less(i, j int) bool {
// pos returns the row position of a row/column pair.
func pos(rowID, columnID uint64) uint64 {
return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth)
return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth)
}
func uriPathToURL(uri *pilosa.URI, path string) url.URL {

62
http/client_test.go
View file

@ -62,42 +62,42 @@ func TestClient_MultiNode(t *testing.T) {
defer s[i].Close()
}
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[0].Holder
e.Node = cluster.Nodes[0]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
return e.Execute(ctx, index, query, shards, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[1].Holder
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
return e.Execute(ctx, index, query, shards, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[2].Holder
e.Node = cluster.Nodes[2]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
return e.Execute(ctx, index, query, shards, opt)
}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN.
shardNums := []uint64{1, 2, 6}
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())`
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
{0, 6, 9, 15, 16, 20},
}
for i, num := range sliceNums {
for i, num := range shardNums {
ownsNum := false
for _, ownNum := range owns[i] {
if ownNum == num {
@ -106,18 +106,18 @@ func TestClient_MultiNode(t *testing.T) {
}
}
if !ownsNum {
t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns)
t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, s[i].Host(), owns)
}
}
baseBit0 := pilosa.SliceWidth * sliceNums[0]
baseBit1 := pilosa.SliceWidth * sliceNums[1]
baseBit2 := pilosa.SliceWidth * sliceNums[2]
baseBit0 := pilosa.ShardWidth * shardNums[0]
baseBit1 := pilosa.ShardWidth * shardNums[1]
baseBit2 := pilosa.ShardWidth * shardNums[2]
maxSlice := uint64(0)
for _, x := range sliceNums {
if x > maxSlice {
maxSlice = x
maxShard := uint64(0)
for _, x := range shardNums {
if x > maxShard {
maxShard = x
}
}
@ -145,9 +145,9 @@ func TestClient_MultiNode(t *testing.T) {
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache()
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache()
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[0]).RecalculateCache()
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[1]).RecalculateCache()
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[2]).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
@ -165,18 +165,18 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatal(err)
}
// Check the results before every node has the correct max slice value.
// Check the results before every node has the correct max shard value.
pairs := result.Results[0].Pairs
for _, pair := range pairs {
if pair.ID == 22 && pair.Count != 3 {
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
t.Fatalf("Invalid Cluster wide MaxShard prevents accurate calculation of %s", pair)
}
}
// Set max slice to correct value.
hldr[0].Index("i").SetRemoteMaxSlice(maxSlice)
hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
// Set max shard to correct value.
hldr[0].Index("i").SetRemoteMaxShard(maxShard)
hldr[1].Index("i").SetRemoteMaxShard(maxShard)
hldr[2].Index("i").SetRemoteMaxShard(maxShard)
result, err = client[0].Query(context.Background(), "i", queryRequest)
if err != nil {
@ -219,7 +219,7 @@ func TestClient_MultiNode(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_Import(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
host := cmd.URL()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -249,7 +249,7 @@ func TestClient_Import(t *testing.T) {
// Ensure client can bulk import value data.
func TestClient_ImportValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
host := cmd.URL()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -321,14 +321,14 @@ func TestClient_ImportValue(t *testing.T) {
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
// Set a bit on a different shard.
hldr.SetBit("i", "f", 0, 1)
c := MustNewClient(cmd.URL(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)

66
http/handler.go
View file

@ -156,13 +156,13 @@ func (h *Handler) Close() error {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "shard")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -195,7 +195,7 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
@ -385,20 +385,20 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
}
// handleGetSlicesMax handles GET /schema requests.
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
// handleGetShardsMax handles GET /shards/max requests.
func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.API.MaxSlices(r.Context()),
if err := json.NewEncoder(w).Encode(getShardsMaxResponse{
Standard: h.API.MaxShards(r.Context()),
}); err != nil {
h.Logger.Printf("write slices-max response error: %s", err)
h.Logger.Printf("write shards-max response error: %s", err)
}
}
type getSlicesMaxResponse struct {
type getShardsMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
}
@ -839,15 +839,15 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
}
query := string(buf)
// Parse list of slices.
slices, err := parseUint64Slice(q.Get("slices"))
// Parse list of shards.
shards, err := parseUint64Slice(q.Get("shards"))
if err != nil {
return nil, errors.New("invalid slice argument")
return nil, errors.New("invalid shard argument")
}
return &pilosa.QueryRequest{
Query: query,
Slices: slices,
Shards: shards,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
ExcludeColumns: q.Get("excludeColumns") == "true",
@ -908,7 +908,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
fallthrough
case pilosa.ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case pilosa.ErrClusterDoesNotOwnSlice:
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -961,7 +961,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
fallthrough
case pilosa.ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case pilosa.ErrClusterDoesNotOwnSlice:
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -998,17 +998,17 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
index, field := q.Get("index"), q.Get("field")
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
shard, err := strconv.ParseUint(q.Get("shard"), 10, 64)
if err != nil {
http.Error(w, "invalid slice", http.StatusBadRequest)
http.Error(w, "invalid shard", http.StatusBadRequest)
return
}
if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil {
if err = h.API.ExportCSV(r.Context(), index, field, shard, w); err != nil {
switch errors.Cause(err) {
case pilosa.ErrFragmentNotFound:
break
case pilosa.ErrClusterDoesNotOwnSlice:
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -1026,15 +1026,15 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
q := r.URL.Query()
index := q.Get("index")
// Read slice parameter.
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
// Read shard parameter.
shard, err := strconv.ParseUint(q.Get("shard"), 10, 64)
if err != nil {
http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest)
http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest)
return
}
// Retrieve fragment owner nodes.
nodes, err := h.API.SliceNodes(r.Context(), index, slice)
nodes, err := h.API.ShardNodes(r.Context(), index, shard)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -1072,15 +1072,15 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
// Read slice parameter.
// Read shard parameter.
q := r.URL.Query()
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
shard, err := strconv.ParseUint(q.Get("shard"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
http.Error(w, "shard required", http.StatusBadRequest)
return
}
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice)
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard)
if err != nil {
if errors.Cause(err) == pilosa.ErrFragmentNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
@ -1131,7 +1131,7 @@ const (
func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest {
req := &pilosa.QueryRequest{
Query: pb.Query,
Slices: pb.Slices,
Shards: pb.Shards,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,

51
http/translator_test.go
View file

@ -15,6 +15,17 @@ import (
"github.com/pilosa/pilosa/test"
)
func newMockReadCloser() *mock.ReadCloser {
return &mock.ReadCloser{
ReadFunc: func(p []byte) (int, error) {
return 0, io.EOF
},
CloseFunc: func() error {
return nil
},
}
}
func TestTranslateStore_Reader(t *testing.T) {
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
@ -37,9 +48,9 @@ func TestTranslateStore_Reader(t *testing.T) {
return 0, nil
}
}
var closeInvoked bool
closeInvoked := make(chan struct{})
mrc.CloseFunc = func() error {
closeInvoked = true
close(closeInvoked)
return nil
}
@ -55,19 +66,11 @@ func TestTranslateStore_Reader(t *testing.T) {
}
return &mrc, nil
}
mrc2 := mock.ReadCloser{
ReadFunc: func(p []byte) (int, error) {
return 0, io.EOF
},
CloseFunc: func() error {
return nil
},
}
return &mrc2, nil
return newMockReadCloser(), nil
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
@ -85,8 +88,11 @@ func TestTranslateStore_Reader(t *testing.T) {
t.Fatal(err)
}
if !closeInvoked {
select {
case <-time.NewTimer(time.Millisecond * 100).C:
t.Fatal("expected server close")
case <-closeInvoked:
return
}
})
@ -100,19 +106,22 @@ func TestTranslateStore_Reader(t *testing.T) {
<-done
return 0, io.EOF
}
var closeInvoked bool
closeInvoked := make(chan struct{})
mrc.CloseFunc = func() error {
closeInvoked = true
close(closeInvoked)
return nil
}
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return &mrc, nil
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
defer close(done)
@ -126,9 +135,11 @@ func TestTranslateStore_Reader(t *testing.T) {
// Cancel the context and check if server is closed.
cancel()
time.Sleep(100 * time.Millisecond)
if !closeInvoked {
t.Fatal("expected server-side close")
select {
case <-time.NewTimer(time.Millisecond * 100).C:
t.Fatal("expected server close")
case <-closeInvoked:
return
}
})
})
@ -141,7 +152,7 @@ func TestTranslateStore_Reader(t *testing.T) {
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
_, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0)

26
index.go
View file

@ -38,8 +38,8 @@ type Index struct {
// Fields by name.
fields map[string]*Field
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
// Max shard on any node in the cluster, according to this node.
remoteMaxShard uint64
NewAttrStore func(string) AttrStore
@ -64,7 +64,7 @@ func NewIndex(path, name string) (*Index, error) {
name: name,
fields: make(map[string]*Field),
remoteMaxSlice: 0,
remoteMaxShard: 0,
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
@ -210,30 +210,30 @@ func (i *Index) Close() error {
return nil
}
// MaxSlice returns the max slice in the index according to this node.
func (i *Index) MaxSlice() uint64 {
// MaxShard returns the max shard in the index according to this node.
func (i *Index) MaxShard() uint64 {
if i == nil {
return 0
}
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxSlice
max := i.remoteMaxShard
for _, f := range i.fields {
if slice := f.MaxSlice(); slice > max {
max = slice
if shard := f.MaxShard(); shard > max {
max = shard
}
}
i.Stats.Gauge("maxSlice", float64(max), 1.0)
i.Stats.Gauge("maxShard", float64(max), 1.0)
return max
}
// SetRemoteMaxSlice sets the remote max slice value received from another node.
func (i *Index) SetRemoteMaxSlice(newmax uint64) {
// SetRemoteMaxShard sets the remote max shard value received from another node.
func (i *Index) SetRemoteMaxShard(newmax uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxSlice = newmax
i.remoteMaxShard = newmax
}
// FieldPath returns the path to a field in the index.
@ -427,7 +427,7 @@ func hasTime(a []*time.Time) bool {
type importKey struct {
View string
Slice uint64
Shard uint64
}
type importData struct {

4
index_test.go
View file

@ -23,8 +23,8 @@ import (
"github.com/pilosa/pilosa/test"
)
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// ShardWidth is a helper reference to use when testing.
const ShardWidth = pilosa.ShardWidth
// Ensure index can open and retrieve a field.
func TestIndex_CreateFieldIfNotExists(t *testing.T) {

408
internal/private.pb.go
View file

@ -1,5 +1,6 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// Code generated by protoc-gen-gogo.
// source: private.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -14,8 +15,8 @@
BlockDataRequest
BlockDataResponse
Cache
MaxSlices
CreateSliceMessage
MaxShards
CreateShardMessage
DeleteIndexMessage
CreateIndexMessage
CreateFieldMessage
@ -159,7 +160,7 @@ type BlockDataRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"`
Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"`
Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"`
Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"`
}
@ -189,9 +190,9 @@ func (m *BlockDataRequest) GetView() string {
return ""
}
func (m *BlockDataRequest) GetSlice() uint64 {
func (m *BlockDataRequest) GetShard() uint64 {
if m != nil {
return m.Slice
return m.Shard
}
return 0
}
@ -243,42 +244,42 @@ func (m *Cache) GetIDs() []uint64 {
return nil
}
type MaxSlices struct {
type MaxShards struct {
Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (m *MaxSlices) Reset() { *m = MaxSlices{} }
func (m *MaxSlices) String() string { return proto.CompactTextString(m) }
func (*MaxSlices) ProtoMessage() {}
func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} }
func (m *MaxShards) Reset() { *m = MaxShards{} }
func (m *MaxShards) String() string { return proto.CompactTextString(m) }
func (*MaxShards) ProtoMessage() {}
func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} }
func (m *MaxSlices) GetStandard() map[string]uint64 {
func (m *MaxShards) GetStandard() map[string]uint64 {
if m != nil {
return m.Standard
}
return nil
}
type CreateSliceMessage struct {
type CreateShardMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"`
}
func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} }
func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) }
func (*CreateSliceMessage) ProtoMessage() {}
func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} }
func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} }
func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) }
func (*CreateShardMessage) ProtoMessage() {}
func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} }
func (m *CreateSliceMessage) GetIndex() string {
func (m *CreateShardMessage) GetIndex() string {
if m != nil {
return m.Index
}
return ""
}
func (m *CreateSliceMessage) GetSlice() uint64 {
func (m *CreateShardMessage) GetShard() uint64 {
if m != nil {
return m.Slice
return m.Shard
}
return 0
}
@ -565,7 +566,7 @@ func (m *NodeEventMessage) GetNode() *Node {
type NodeStatus struct {
Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"`
MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"`
MaxShards *MaxShards `protobuf:"bytes,2,opt,name=MaxShards" json:"MaxShards,omitempty"`
Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"`
}
@ -581,9 +582,9 @@ func (m *NodeStatus) GetNode() *Node {
return nil
}
func (m *NodeStatus) GetMaxSlices() *MaxSlices {
func (m *NodeStatus) GetMaxShards() *MaxShards {
if m != nil {
return m.MaxSlices
return m.MaxShards
}
return nil
}
@ -792,7 +793,7 @@ type ResizeSource struct {
Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"`
View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"`
Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"`
Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"`
}
func (m *ResizeSource) Reset() { *m = ResizeSource{} }
@ -828,9 +829,9 @@ func (m *ResizeSource) GetView() string {
return ""
}
func (m *ResizeSource) GetSlice() uint64 {
func (m *ResizeSource) GetShard() uint64 {
if m != nil {
return m.Slice
return m.Shard
}
return 0
}
@ -940,8 +941,8 @@ func init() {
proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest")
proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse")
proto.RegisterType((*Cache)(nil), "internal.Cache")
proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices")
proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage")
proto.RegisterType((*MaxShards)(nil), "internal.MaxShards")
proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage")
proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage")
proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage")
proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage")
@ -1111,10 +1112,10 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Block))
}
if m.Slice != 0 {
if m.Shard != 0 {
dAtA[i] = 0x20
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
i = encodeVarintPrivate(dAtA, i, uint64(m.Shard))
}
if len(m.View) > 0 {
dAtA[i] = 0x2a
@ -1212,7 +1213,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *MaxSlices) Marshal() (dAtA []byte, err error) {
func (m *MaxShards) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -1222,7 +1223,7 @@ func (m *MaxSlices) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) {
func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
@ -1246,7 +1247,7 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) {
func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -1256,7 +1257,7 @@ func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
@ -1267,10 +1268,10 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if m.Slice != 0 {
if m.Shard != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
i = encodeVarintPrivate(dAtA, i, uint64(m.Shard))
}
return i, nil
}
@ -1685,11 +1686,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) {
}
i += n12
}
if m.MaxSlices != nil {
if m.MaxShards != nil {
dAtA[i] = 0x12
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size()))
n13, err := m.MaxSlices.MarshalTo(dAtA[i:])
i = encodeVarintPrivate(dAtA, i, uint64(m.MaxShards.Size()))
n13, err := m.MaxShards.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
@ -1980,10 +1981,10 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(len(m.View)))
i += copy(dAtA[i:], m.View)
}
if m.Slice != 0 {
if m.Shard != 0 {
dAtA[i] = 0x28
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
i = encodeVarintPrivate(dAtA, i, uint64(m.Shard))
}
return i, nil
}
@ -2140,6 +2141,24 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Private(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Private(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -2212,8 +2231,8 @@ func (m *BlockDataRequest) Size() (n int) {
if m.Block != 0 {
n += 1 + sovPrivate(uint64(m.Block))
}
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
if m.Shard != 0 {
n += 1 + sovPrivate(uint64(m.Shard))
}
l = len(m.View)
if l > 0 {
@ -2255,7 +2274,7 @@ func (m *Cache) Size() (n int) {
return n
}
func (m *MaxSlices) Size() (n int) {
func (m *MaxShards) Size() (n int) {
var l int
_ = l
if len(m.Standard) > 0 {
@ -2269,15 +2288,15 @@ func (m *MaxSlices) Size() (n int) {
return n
}
func (m *CreateSliceMessage) Size() (n int) {
func (m *CreateShardMessage) Size() (n int) {
var l int
_ = l
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
if m.Shard != 0 {
n += 1 + sovPrivate(uint64(m.Shard))
}
return n
}
@ -2454,8 +2473,8 @@ func (m *NodeStatus) Size() (n int) {
l = m.Node.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.MaxSlices != nil {
l = m.MaxSlices.Size()
if m.MaxShards != nil {
l = m.MaxShards.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.Schema != nil {
@ -2591,8 +2610,8 @@ func (m *ResizeSource) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
if m.Shard != 0 {
n += 1 + sovPrivate(uint64(m.Shard))
}
return n
}
@ -3140,9 +3159,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
}
m.Slice = 0
m.Shard = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
@ -3152,7 +3171,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
m.Shard |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -3493,7 +3512,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *MaxSlices) Unmarshal(dAtA []byte) error {
func (m *MaxShards) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -3516,10 +3535,10 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group")
return fmt.Errorf("proto: MaxShards: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: MaxShards: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
@ -3548,14 +3567,51 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
var keykey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
keykey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey := string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
if m.Standard == nil {
m.Standard = make(map[string]uint64)
}
var mapkey string
var mapvalue uint64
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
if iNdEx < postIndex {
var valuekey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
@ -3565,69 +3621,31 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
valuekey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
var mapvalue uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
} else {
iNdEx = entryPreIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
m.Standard[mapkey] = mapvalue
} else {
var mapvalue uint64
m.Standard[mapkey] = mapvalue
}
m.Standard[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
@ -3650,7 +3668,7 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
func (m *CreateShardMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -3673,10 +3691,10 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group")
return fmt.Errorf("proto: CreateShardMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: CreateShardMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
@ -3710,9 +3728,9 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
}
m.Slice = 0
m.Shard = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
@ -3722,7 +3740,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
m.Shard |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -5053,7 +5071,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field MaxShards", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -5077,10 +5095,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.MaxSlices == nil {
m.MaxSlices = &MaxSlices{}
if m.MaxShards == nil {
m.MaxShards = &MaxShards{}
}
if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
if err := m.MaxShards.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -6080,9 +6098,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
}
m.Slice = 0
m.Shard = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
@ -6092,7 +6110,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
m.Shard |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -6681,70 +6699,70 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1028 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35,
0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50,
0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0,
0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05,
0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4,
0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb,
0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0,
0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04,
0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7,
0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8,
0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05,
0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b,
0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74,
0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0,
0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79,
0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13,
0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10,
0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7,
0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac,
0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d,
0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d,
0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14,
0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a,
0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85,
0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27,
0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8,
0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22,
0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd,
0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18,
0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0,
0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac,
0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf,
0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c,
0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8,
0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33,
0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2,
0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c,
0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4,
0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32,
0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52,
0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09,
0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98,
0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00,
0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4,
0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89,
0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f,
0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa,
0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f,
0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc,
0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa,
0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d,
0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c,
0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b,
0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97,
0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70,
0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f,
0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66,
0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63,
0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf,
0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99,
0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a,
0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab,
0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1,
0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21,
0x0a, 0x0b, 0x00, 0x00,
// 1027 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0x45,
0x18, 0x66, 0xbd, 0x6b, 0xc7, 0x7e, 0x53, 0x87, 0x64, 0x0a, 0x61, 0x8b, 0x50, 0x6a, 0x46, 0x95,
0x1a, 0x7a, 0x88, 0x4a, 0x7b, 0xe1, 0xab, 0x52, 0x14, 0x3b, 0xc0, 0x02, 0x09, 0x30, 0x9b, 0xf4,
0xd6, 0xc3, 0xd4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0x96, 0xdd, 0xd9, 0x24, 0xee, 0x81, 0x2b, 0x5c,
0xb8, 0x23, 0x7e, 0x09, 0x3f, 0x81, 0x23, 0x3f, 0x01, 0x85, 0x3f, 0x82, 0xe6, 0x9d, 0xd9, 0x8f,
0xc4, 0x4e, 0x53, 0x85, 0xde, 0xe6, 0xfd, 0x7e, 0xe6, 0xfd, 0x9a, 0x81, 0x7e, 0x9a, 0x45, 0x27,
0x5c, 0x89, 0xad, 0x34, 0x93, 0x4a, 0x92, 0x6e, 0x94, 0x28, 0x91, 0x25, 0x3c, 0xa6, 0x77, 0xa1,
0x17, 0x24, 0x13, 0x71, 0xb6, 0x27, 0x14, 0x27, 0x04, 0xbc, 0x6f, 0xc5, 0x2c, 0xf7, 0xdd, 0x81,
0xb3, 0xd9, 0x65, 0x78, 0xa6, 0x7f, 0x3a, 0x70, 0xeb, 0xcb, 0x48, 0xc4, 0x93, 0xef, 0x53, 0x15,
0xc9, 0x24, 0x27, 0x1f, 0x40, 0x6f, 0xc8, 0xc7, 0x47, 0xe2, 0x60, 0x96, 0x0a, 0xd4, 0xec, 0xb1,
0x9a, 0x51, 0x49, 0xc3, 0xe8, 0xa5, 0xf0, 0xbd, 0x81, 0xb3, 0xd9, 0x67, 0x35, 0x83, 0x0c, 0x60,
0xf9, 0x20, 0x9a, 0x8a, 0x1f, 0x0b, 0x9e, 0xa8, 0x62, 0xea, 0xb7, 0xd1, 0xba, 0xc9, 0xd2, 0x10,
0xd0, 0x71, 0x17, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xee, 0x45, 0x89, 0xdf, 0x1b, 0x38, 0x9b, 0x2e,
0xd3, 0x47, 0xe4, 0xf0, 0x33, 0x1f, 0x2c, 0x87, 0x9f, 0x55, 0xd0, 0x97, 0x1b, 0xd0, 0x29, 0xac,
0x04, 0xd3, 0x54, 0x66, 0x8a, 0x89, 0x3c, 0x95, 0x49, 0x8e, 0x9e, 0x76, 0xb3, 0xcc, 0x77, 0xd0,
0xb9, 0x3e, 0xd2, 0x9f, 0x61, 0x75, 0x27, 0x96, 0xe3, 0xe3, 0x11, 0x57, 0x9c, 0x89, 0x9f, 0x0a,
0x91, 0x2b, 0xf2, 0x0e, 0xb4, 0x31, 0x27, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x79, 0xf0, 0x5b, 0x86,
0x8b, 0x84, 0xe6, 0xa2, 0x3d, 0x66, 0xc2, 0x63, 0x86, 0xd0, 0xdc, 0xf0, 0x88, 0x67, 0x13, 0xcc,
0x80, 0xc7, 0x0c, 0xa1, 0x31, 0x3e, 0x8d, 0xc4, 0xa9, 0xbd, 0x36, 0x9e, 0x69, 0x00, 0x6b, 0x8d,
0xf8, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, 0x59,
0x0a, 0x93, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x51, 0x0b, 0x45, 0x35, 0x83, 0xde, 0x81, 0x36, 0x66,
0x5a, 0xdf, 0xb2, 0xb6, 0xd5, 0x47, 0xfa, 0x8b, 0x03, 0xbd, 0x3d, 0x7e, 0x86, 0x30, 0x72, 0xf2,
0x04, 0xba, 0xa1, 0xe2, 0xc9, 0x44, 0x03, 0xd4, 0x4a, 0xcb, 0x8f, 0x3e, 0xdc, 0x2a, 0x1b, 0x62,
0xab, 0x52, 0xdb, 0x2a, 0x75, 0x76, 0x13, 0x95, 0xcd, 0x58, 0x65, 0xf2, 0xfe, 0xe7, 0xd0, 0xbf,
0x20, 0xd2, 0xf1, 0x8e, 0xc5, 0xac, 0xcc, 0xea, 0xb1, 0x98, 0xe9, 0xfb, 0x9f, 0xf0, 0xb8, 0x10,
0x98, 0x2b, 0x8f, 0x19, 0xe2, 0xb3, 0xd6, 0x27, 0x0e, 0xdd, 0x06, 0x32, 0xcc, 0x04, 0x57, 0x02,
0x83, 0xec, 0x89, 0x3c, 0xe7, 0x2f, 0xc4, 0xd5, 0x19, 0x37, 0x59, 0x6c, 0x35, 0xb2, 0x48, 0x1f,
0x00, 0x19, 0x89, 0x58, 0x28, 0x61, 0xfb, 0xf6, 0x15, 0x1e, 0x68, 0x58, 0x46, 0xbb, 0x5e, 0x97,
0xdc, 0x07, 0x4f, 0x0f, 0x01, 0x06, 0x5b, 0x7e, 0x74, 0xbb, 0xce, 0x48, 0x35, 0x1f, 0x0c, 0x15,
0x68, 0x5c, 0x3a, 0xc5, 0x0e, 0xb8, 0xf6, 0x0a, 0x0b, 0x9a, 0xe6, 0x81, 0x0d, 0xe5, 0x62, 0xa8,
0xf5, 0x3a, 0x54, 0x73, 0xd0, 0x6c, 0xb4, 0xed, 0xf2, 0xba, 0x37, 0x8d, 0x46, 0x9f, 0x59, 0xae,
0xee, 0xbf, 0x7d, 0x3e, 0x15, 0xd6, 0x06, 0xcf, 0x15, 0x94, 0xd6, 0xf5, 0x50, 0xb4, 0x7b, 0xdd,
0xb3, 0x7a, 0x3f, 0xb8, 0xda, 0x3d, 0x12, 0xf4, 0x31, 0x74, 0xc2, 0xf1, 0x91, 0x98, 0x72, 0xf2,
0x11, 0x2c, 0x21, 0x0e, 0x91, 0xdb, 0xb6, 0x7a, 0xfb, 0x52, 0x12, 0x59, 0x29, 0xa7, 0x23, 0x8b,
0x7f, 0x21, 0xa6, 0xfb, 0xd0, 0xc1, 0xe8, 0xb9, 0xef, 0x5d, 0x76, 0x83, 0x7c, 0x66, 0xc5, 0x74,
0x17, 0xdc, 0x43, 0x16, 0xe8, 0x71, 0x41, 0x04, 0xa5, 0x17, 0x4b, 0x69, 0xdf, 0x5f, 0xcb, 0x5c,
0xd9, 0x6c, 0xe0, 0x59, 0xf3, 0x7e, 0x90, 0x99, 0xc2, 0xd4, 0xf7, 0x19, 0x9e, 0xe9, 0x33, 0xf0,
0xf6, 0xe5, 0x44, 0x90, 0x15, 0x68, 0x05, 0x23, 0xeb, 0xa3, 0x15, 0x8c, 0xc8, 0x5d, 0x74, 0x6f,
0x53, 0xd3, 0xaf, 0x41, 0x1c, 0xb2, 0x80, 0x61, 0xe0, 0x7b, 0xd0, 0x0f, 0xf2, 0xa1, 0x94, 0xd9,
0x24, 0x4a, 0xb8, 0x92, 0x99, 0x5d, 0x9c, 0x17, 0x99, 0x74, 0x1b, 0x56, 0xb5, 0xfb, 0x50, 0x71,
0x25, 0xca, 0xfa, 0xad, 0x43, 0x47, 0xf3, 0xaa, 0x70, 0x96, 0xc2, 0x96, 0xd7, 0x7a, 0x65, 0x05,
0x91, 0xa0, 0xdf, 0x19, 0x0f, 0xbb, 0x27, 0x22, 0x51, 0x8d, 0x0e, 0x40, 0x1a, 0x1d, 0xf4, 0x99,
0x21, 0x08, 0x35, 0x57, 0xb1, 0x98, 0x57, 0x6a, 0xcc, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00,
0x4a, 0x40, 0x45, 0x5e, 0x99, 0x38, 0x57, 0x9b, 0x90, 0x8f, 0x1b, 0xeb, 0x63, 0x7e, 0x40, 0x2a,
0x11, 0x6b, 0x2c, 0x99, 0xcd, 0xb2, 0x2d, 0x6c, 0x97, 0xaf, 0xd6, 0xfa, 0x86, 0x6f, 0xcb, 0xc4,
0x69, 0x04, 0xfd, 0x61, 0x5c, 0xe4, 0x4a, 0x64, 0x16, 0x91, 0x5e, 0x73, 0x86, 0x51, 0xe5, 0xa7,
0x66, 0x2c, 0x4e, 0x11, 0xb9, 0x07, 0x6d, 0x8d, 0xd4, 0xf4, 0xe6, 0xfc, 0x35, 0x8c, 0x90, 0x3e,
0x85, 0xee, 0x4e, 0x18, 0x7c, 0x95, 0xc9, 0x22, 0x5d, 0xd8, 0x79, 0xe5, 0xeb, 0xd3, 0x9a, 0x7f,
0x7d, 0xdc, 0xb9, 0xd7, 0xc7, 0xab, 0x5e, 0x1f, 0x1a, 0xc2, 0x9a, 0x59, 0x09, 0x7a, 0x24, 0x6e,
0xb2, 0x11, 0xca, 0xa7, 0xc1, 0x6d, 0x3c, 0x0d, 0x21, 0xac, 0x99, 0xc9, 0x7f, 0x93, 0x4e, 0xff,
0x68, 0xc1, 0x1a, 0x13, 0x79, 0xf4, 0x52, 0x04, 0x49, 0xae, 0xb2, 0x62, 0xac, 0x07, 0x5c, 0xdb,
0x7f, 0x23, 0x9f, 0xdb, 0x6c, 0xbb, 0xcc, 0x10, 0xaf, 0xd3, 0x4c, 0xe4, 0x21, 0x2c, 0x5f, 0x1e,
0x80, 0x79, 0xd5, 0xa6, 0x0a, 0x79, 0x08, 0x4b, 0xa1, 0x2c, 0xb2, 0xb1, 0x28, 0xc7, 0xbb, 0xb1,
0x74, 0x0c, 0x32, 0x23, 0x66, 0xa5, 0x5a, 0xa3, 0x95, 0xda, 0xaf, 0x6e, 0x25, 0xf2, 0xe4, 0x52,
0x2b, 0xf9, 0x1d, 0x34, 0x78, 0xaf, 0x36, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0xaf, 0x0e, 0xdc,
0x6a, 0x42, 0x78, 0xad, 0xd9, 0xa8, 0x2a, 0xd2, 0x5a, 0x58, 0x11, 0x77, 0x51, 0x45, 0xbc, 0xba,
0x22, 0xf5, 0x2b, 0xd7, 0x6e, 0xbe, 0x72, 0xc7, 0x70, 0x67, 0xae, 0x4c, 0x43, 0x39, 0x4d, 0x75,
0x3f, 0xfc, 0x8f, 0x72, 0xe9, 0xad, 0x91, 0x65, 0xb6, 0x50, 0x3d, 0x66, 0x08, 0xfa, 0x29, 0xbc,
0x1b, 0x0a, 0xd5, 0x28, 0x52, 0xd9, 0x6d, 0x03, 0x70, 0xf7, 0xc5, 0xe9, 0x15, 0xd7, 0xd7, 0x22,
0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0x23, 0xeb, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6,
0xf2, 0xc5, 0xec, 0x9a, 0xa9, 0xf7, 0x61, 0xc9, 0xac, 0x48, 0xf3, 0xf1, 0xe9, 0xb1, 0x92, 0xa4,
0xb7, 0x75, 0x43, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, 0x43, 0xff, 0x80, 0xf2, 0x9d, 0xd5, 0xbf,
0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0x7f, 0xce, 0x37, 0x9c, 0xdf, 0xff, 0xdd, 0x78, 0xeb,
0x79, 0x07, 0x7f, 0xbe, 0x8f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x39, 0x2f, 0x93, 0x68, 0x0a,
0x0b, 0x00, 0x00,
}

12
internal/private.proto
View file

@ -24,7 +24,7 @@ message BlockDataRequest {
string Index = 1;
string Field = 2;
string View = 5;
uint64 Slice = 4;
uint64 Shard = 4;
uint64 Block = 3;
}
@ -37,13 +37,13 @@ message Cache {
repeated uint64 IDs = 1;
}
message MaxSlices {
message MaxShards {
map<string, uint64> Standard = 1;
}
message CreateSliceMessage {
message CreateShardMessage {
string Index = 1;
uint64 Slice = 2;
uint64 Shard = 2;
}
message DeleteIndexMessage {
@ -105,7 +105,7 @@ message NodeEventMessage {
message NodeStatus {
Node Node = 1;
MaxSlices MaxSlices = 2;
MaxShards MaxShards = 2;
Schema Schema = 3;
}
@ -148,7 +148,7 @@ message ResizeSource {
string Index = 2;
string Field = 3;
string View = 4;
uint64 Slice = 5;
uint64 Shard = 5;
}
message ResizeInstructionComplete {

187
internal/public.pb.go
View file

@ -1,5 +1,6 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// Code generated by protoc-gen-gogo.
// source: public.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import encoding_binary "encoding/binary"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
@ -268,7 +267,7 @@ func (m *AttrMap) GetAttrs() []*Attr {
type QueryRequest struct {
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"`
@ -287,9 +286,9 @@ func (m *QueryRequest) GetQuery() string {
return ""
}
func (m *QueryRequest) GetSlices() []uint64 {
func (m *QueryRequest) GetShards() []uint64 {
if m != nil {
return m.Slices
return m.Shards
}
return nil
}
@ -413,7 +412,7 @@ func (m *QueryResult) GetChanged() bool {
type ImportRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"`
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"`
@ -440,9 +439,9 @@ func (m *ImportRequest) GetField() string {
return ""
}
func (m *ImportRequest) GetSlice() uint64 {
func (m *ImportRequest) GetShard() uint64 {
if m != nil {
return m.Slice
return m.Shard
}
return 0
}
@ -485,7 +484,7 @@ func (m *ImportRequest) GetTimestamps() []int64 {
type ImportValueRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"`
Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
@ -510,9 +509,9 @@ func (m *ImportValueRequest) GetField() string {
return ""
}
func (m *ImportValueRequest) GetSlice() uint64 {
func (m *ImportValueRequest) GetShard() uint64 {
if m != nil {
return m.Slice
return m.Shard
}
return 0
}
@ -800,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
i += 8
i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue))))
}
return i, nil
}
@ -857,10 +855,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Query)))
i += copy(dAtA[i:], m.Query)
}
if len(m.Slices) > 0 {
dAtA4 := make([]byte, len(m.Slices)*10)
if len(m.Shards) > 0 {
dAtA4 := make([]byte, len(m.Shards)*10)
var j3 int
for _, num := range m.Slices {
for _, num := range m.Shards {
for num >= 1<<7 {
dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -1062,10 +1060,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
if m.Shard != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
i = encodeVarintPublic(dAtA, i, uint64(m.Shard))
}
if len(m.RowIDs) > 0 {
dAtA8 := make([]byte, len(m.RowIDs)*10)
@ -1179,10 +1177,10 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
if m.Shard != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
i = encodeVarintPublic(dAtA, i, uint64(m.Shard))
}
if len(m.ColumnIDs) > 0 {
dAtA14 := make([]byte, len(m.ColumnIDs)*10)
@ -1237,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Public(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Public(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -1378,9 +1394,9 @@ func (m *QueryRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if len(m.Slices) > 0 {
if len(m.Shards) > 0 {
l = 0
for _, e := range m.Slices {
for _, e := range m.Shards {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
@ -1462,8 +1478,8 @@ func (m *ImportRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPublic(uint64(m.Slice))
if m.Shard != 0 {
n += 1 + sovPublic(uint64(m.Shard))
}
if len(m.RowIDs) > 0 {
l = 0
@ -1512,8 +1528,8 @@ func (m *ImportValueRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPublic(uint64(m.Slice))
if m.Shard != 0 {
n += 1 + sovPublic(uint64(m.Shard))
}
if len(m.ColumnIDs) > 0 {
l = 0
@ -2317,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if (iNdEx + 8) > l {
return io.ErrUnexpectedEOF
}
v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
v = uint64(dAtA[iNdEx-8])
v |= uint64(dAtA[iNdEx-7]) << 8
v |= uint64(dAtA[iNdEx-6]) << 16
v |= uint64(dAtA[iNdEx-5]) << 24
v |= uint64(dAtA[iNdEx-4]) << 32
v |= uint64(dAtA[iNdEx-3]) << 40
v |= uint64(dAtA[iNdEx-2]) << 48
v |= uint64(dAtA[iNdEx-1]) << 56
m.FloatValue = float64(math.Float64frombits(v))
default:
iNdEx = preIndex
@ -2497,7 +2520,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.Slices = append(m.Slices, v)
m.Shards = append(m.Shards, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
@ -2537,10 +2560,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.Slices = append(m.Slices, v)
m.Shards = append(m.Shards, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shards", wireType)
}
case 3:
if wireType != 0 {
@ -3078,9 +3101,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
}
m.Slice = 0
m.Shard = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -3090,7 +3113,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
m.Shard |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -3449,9 +3472,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType)
}
m.Slice = 0
m.Shard = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -3461,7 +3484,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
m.Shard |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -3748,49 +3771,49 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 699 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e,
0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54,
0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d,
0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73,
0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26,
0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0,
0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe,
0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46,
0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee,
0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3,
0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5,
0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12,
0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b,
0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98,
0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c,
0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf,
0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6,
0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda,
0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa,
0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3,
0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57,
0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2,
0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd,
0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10,
0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3,
0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95,
0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1,
0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59,
0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5,
0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab,
0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb,
0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94,
0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82,
0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9,
0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9,
0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7,
0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1,
0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f,
0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d,
0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66,
0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90,
0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00,
0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00,
// 701 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84,
0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b,
0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0,
0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6,
0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5,
0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8,
0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41,
0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69,
0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b,
0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93,
0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1,
0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3,
0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c,
0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7,
0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22,
0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb,
0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49,
0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75,
0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88,
0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c,
0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d,
0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97,
0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd,
0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44,
0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58,
0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd,
0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1,
0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85,
0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2,
0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6,
0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b,
0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85,
0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84,
0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64,
0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66,
0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58,
0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9,
0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17,
0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09,
0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e,
0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8,
0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc,
0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00,
}

6
internal/public.proto
View file

@ -46,7 +46,7 @@ message AttrMap {
message QueryRequest {
string Query = 1;
repeated uint64 Slices = 2;
repeated uint64 Shards = 2;
bool ColumnAttrs = 3;
bool Remote = 5;
bool ExcludeRowAttrs = 6;
@ -71,7 +71,7 @@ message QueryResult {
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Slice = 3;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated string RowKeys = 7;
@ -82,7 +82,7 @@ message ImportRequest {
message ImportValueRequest {
string Index = 1;
string Field = 2;
uint64 Slice = 3;
uint64 Shard = 3;
repeated uint64 ColumnIDs = 5;
repeated string ColumnKeys = 7;
repeated int64 Values = 6;

4
iterator.go
View file

@ -184,11 +184,11 @@ func NewRoaringIterator(itr *roaring.Iterator) *RoaringIterator {
// Seek moves the cursor to a pair matching bseek/pseek.
// If the pair is not found then it moves to the next pair.
func (itr *RoaringIterator) Seek(bseek, pseek uint64) {
itr.itr.Seek((bseek * SliceWidth) + pseek)
itr.itr.Seek((bseek * ShardWidth) + pseek)
}
// Next returns the next column/row ID pair.
func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) {
v, eof := itr.itr.Next()
return v / SliceWidth, v % SliceWidth, eof
return v / ShardWidth, v % ShardWidth, eof
}

9
mock/mock.go
View file

@ -1,8 +1,11 @@
package mock
import "sync"
type ReadCloser struct {
ReadFunc func(p []byte) (int, error)
CloseFunc func() error
once sync.Once
}
func (rc *ReadCloser) Read(p []byte) (int, error) {
@ -10,5 +13,9 @@ func (rc *ReadCloser) Read(p []byte) (int, error) {
}
func (rc *ReadCloser) Close() error {
return rc.CloseFunc()
var err error = nil
rc.once.Do(func() {
err = rc.CloseFunc()
})
return err
}

2
pilosa.go
View file

@ -56,7 +56,7 @@ var (
ErrQueryRequired = errors.New("query required")
ErrTooManyWrites = errors.New("too many write commands")
ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice")
ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")

24
roaring/roaring_test.go
View file

@ -471,10 +471,10 @@ func TestBitmap_Difference(t *testing.T) {
}
func TestBitmap_Difference2(t *testing.T) {
bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7)
bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1)
bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.ShardWidth+5, pilosa.ShardWidth+7)
bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.ShardWidth+1)
result := bm0.Difference(bm1)
if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) {
if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.ShardWidth + 5, pilosa.ShardWidth + 7}) {
t.Fatalf("unexpected : %v", result.Slice())
}
}
@ -1161,7 +1161,7 @@ func BenchmarkContainerLinear(b *testing.B) {
bm := roaring.NewFileBitmap()
for row := uint64(1); row < NumRows; row++ {
for col := uint64(1); col < NumColums; col++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal))
}
}
}
@ -1172,7 +1172,7 @@ func BenchmarkContainerReverse(b *testing.B) {
bm := roaring.NewFileBitmap()
for row := NumRows - 1; row >= 1; row-- {
for col := NumColums - 1; col >= 1; col-- {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal))
}
}
}
@ -1183,7 +1183,7 @@ func BenchmarkContainerColumn(b *testing.B) {
bm := roaring.NewFileBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < NumRows; row++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal))
}
}
}
@ -1196,8 +1196,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) {
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < middle; row++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal))
bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal))
}
}
}
@ -1209,8 +1209,8 @@ func BenchmarkContainerInsideOut(b *testing.B) {
bm := roaring.NewFileBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row <= middle; row++ {
bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal))
bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal))
}
}
}
@ -1219,7 +1219,7 @@ func BenchmarkContainerInsideOut(b *testing.B) {
func BenchmarkSliceAscending(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewFileBitmap()
for col := uint64(0); col < pilosa.SliceWidth; col++ {
for col := uint64(0); col < pilosa.ShardWidth; col++ {
bm.Add(col)
}
}
@ -1228,7 +1228,7 @@ func BenchmarkSliceAscending(b *testing.B) {
func BenchmarkSliceDescending(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewFileBitmap()
for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- {
for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- {
bm.Add(col)
}
}

44
row.go
View file

@ -157,12 +157,12 @@ func (r *Row) Difference(other *Row) *Row {
// SetBit sets the i-th column of the row.
func (r *Row) SetBit(i uint64) (changed bool) {
return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i)
}
// ClearBit clears the i-th column of the row.
func (r *Row) ClearBit(i uint64) (changed bool) {
s := r.segment(i / SliceWidth)
s := r.segment(i / ShardWidth)
if s == nil {
return false
}
@ -174,24 +174,24 @@ func (r *Row) Segments() []RowSegment {
return r.segments
}
// segment returns a segment for a given slice.
// segment returns a segment for a given shard.
// Returns nil if segment does not exist.
func (r *Row) segment(slice uint64) *RowSegment {
func (r *Row) segment(shard uint64) *RowSegment {
if i := sort.Search(len(r.segments), func(i int) bool {
return r.segments[i].slice >= slice
}); i < len(r.segments) && r.segments[i].slice == slice {
return r.segments[i].shard >= shard
}); i < len(r.segments) && r.segments[i].shard == shard {
return &r.segments[i]
}
return nil
}
func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment {
func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment {
i := sort.Search(len(r.segments), func(i int) bool {
return r.segments[i].slice >= slice
return r.segments[i].shard >= shard
})
// Return exact match.
if i < len(r.segments) && r.segments[i].slice == slice {
if i < len(r.segments) && r.segments[i].shard == shard {
return &r.segments[i]
}
@ -202,7 +202,7 @@ func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment {
}
r.segments[i] = RowSegment{
data: *roaring.NewBitmap(),
slice: slice,
shard: shard,
writable: true,
}
@ -218,7 +218,7 @@ func (r *Row) InvalidateCount() {
// IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased.
func (r *Row) IncrementCount(i uint64) {
seg := r.segment(i / SliceWidth)
seg := r.segment(i / ShardWidth)
if seg != nil {
seg.n++
}
@ -227,7 +227,7 @@ func (r *Row) IncrementCount(i uint64) {
// DecrementCount decrements the row cached counter.
func (r *Row) DecrementCount(i uint64) {
seg := r.segment(i / SliceWidth)
seg := r.segment(i / ShardWidth)
if seg != nil {
if seg.n > 0 {
seg.n--
@ -308,10 +308,10 @@ func Union(rows []*Row) *Row {
// RowSegment holds a subset of a row.
// This could point to a mmapped roaring bitmap or an in-memory bitmap. The
// width of the segment will always match the slice width.
// width of the segment will always match the shard width.
type RowSegment struct {
// Slice this segment belongs to
slice uint64
// Shard this segment belongs to
shard uint64
// Underlying raw bitmap implementation.
// This is an mmapped bitmap if writable is false. Otherwise
@ -345,7 +345,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment {
return &RowSegment{
data: *data,
slice: s.slice,
shard: s.shard,
n: data.Count(),
}
}
@ -356,7 +356,7 @@ func (s *RowSegment) Union(other *RowSegment) *RowSegment {
return &RowSegment{
data: *data,
slice: s.slice,
shard: s.shard,
n: data.Count(),
}
}
@ -367,7 +367,7 @@ func (s *RowSegment) Difference(other *RowSegment) *RowSegment {
return &RowSegment{
data: *data,
slice: s.slice,
shard: s.shard,
n: data.Count(),
}
}
@ -378,7 +378,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
return &RowSegment{
data: *data,
slice: s.slice,
shard: s.shard,
n: data.Count(),
}
}
@ -464,15 +464,15 @@ func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) {
}
// Otherwise determine which is first.
if s0.slice < s1.slice {
if s0.shard < s1.shard {
itr.a0 = itr.a0[1:]
return s0, nil
} else if s0.slice > s1.slice {
} else if s0.shard > s1.shard {
itr.a1 = itr.a1[1:]
return s1, nil
}
// Return both if slices are equal.
// Return both if shards are equal.
itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:]
return s0, s1
}

20
row_test.go
View file

@ -30,7 +30,7 @@ func TestRow_Merge(t *testing.T) {
exp uint64
}{
{
r1: pilosa.NewRow(1, 2, 3, SliceWidth+1, 2*SliceWidth),
r1: pilosa.NewRow(1, 2, 3, ShardWidth+1, 2*ShardWidth),
r2: pilosa.NewRow(3, 4, 5),
exp: 7,
},
@ -56,9 +56,9 @@ func TestRow_Merge(t *testing.T) {
// Ensure a row can Xor'ed
func TestRow_Xor(t *testing.T) {
r1 := pilosa.NewRow(0, 1, SliceWidth)
r2 := pilosa.NewRow(0, 2*SliceWidth)
exp := []uint64{1, SliceWidth, 2 * SliceWidth}
r1 := pilosa.NewRow(0, 1, ShardWidth)
r2 := pilosa.NewRow(0, 2*ShardWidth)
exp := []uint64{1, ShardWidth, 2 * ShardWidth}
res := r1.Xor(r2)
if res.Count() != 3 {
@ -78,9 +78,9 @@ func TestRow_Xor(t *testing.T) {
}
func TestRow_Union_Segment(t *testing.T) {
r1 := pilosa.NewRow(0, 1, SliceWidth)
r2 := pilosa.NewRow(0, 2*SliceWidth)
exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth}
r1 := pilosa.NewRow(0, 1, ShardWidth)
r2 := pilosa.NewRow(0, 2*ShardWidth)
exp := []uint64{0, 1, ShardWidth, 2 * ShardWidth}
res := r1.Union(r2)
if res.Count() != 4 {
@ -99,9 +99,9 @@ func TestRow_Union_Segment(t *testing.T) {
}
func TestRow_Difference_Segment(t *testing.T) {
r1 := pilosa.NewRow(0, 1, SliceWidth)
r2 := pilosa.NewRow(0, 2*SliceWidth)
exp := []uint64{1, SliceWidth}
r1 := pilosa.NewRow(0, 1, ShardWidth)
r2 := pilosa.NewRow(0, 2*ShardWidth)
exp := []uint64{1, ShardWidth}
res := r1.Difference(r2)
if res.Count() != 2 {

22
server.go
View file

@ -411,12 +411,12 @@ func (s *Server) monitorAntiEntropy() {
// ReceiveMessage represents an implementation of BroadcastHandler.
func (s *Server) ReceiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.CreateSliceMessage:
case *internal.CreateShardMessage:
idx := s.holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
idx.SetRemoteMaxSlice(obj.Slice)
idx.SetRemoteMaxShard(obj.Shard)
case *internal.CreateIndexMessage:
opt := IndexOptions{}
_, err := s.holder.CreateIndex(obj.Index, opt)
@ -536,7 +536,7 @@ func (s *Server) Node() *Node {
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice by Index
// - MaxShard by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
@ -549,7 +549,7 @@ func (s *Server) LocalStatus() (proto.Message, error) {
ns := internal.NodeStatus{
Node: EncodeNode(s.cluster.Node),
MaxSlices: s.holder.EncodeMaxSlices(),
MaxShards: s.holder.EncodeMaxShards(),
Schema: s.holder.EncodeSchema(),
}
@ -587,19 +587,19 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
return errors.Wrap(err, "applying schema")
}
// Sync maxSlices.
oldmaxslices := s.holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
// Sync maxShards.
oldmaxshards := s.holder.MaxShards()
for index, newMax := range ns.MaxShards.Standard {
localIndex := s.holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
// indexes should be created and synced prior to shard creation
if localIndex == nil {
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMax > oldmaxslices[index] {
oldmaxslices[index] = newMax
localIndex.SetRemoteMaxSlice(newMax)
if newMax > oldmaxshards[index] {
oldmaxshards[index] = newMax
localIndex.SetRemoteMaxShard(newMax)
}
}

54
server/cluster_test.go
View file

@ -31,7 +31,7 @@ import (
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
ms := test.MustRunMainWithCluster(t, 2)
ms := test.MustRunCluster(t, 2)
m0, m1 := ms[0], ms[1]
defer m0.Close()
defer m1.Close()
@ -95,28 +95,28 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
// Make sure node0 knows about the latest MaxSlice.
maxSlices0, err := client0.MaxSliceByIndex(context.Background())
// Make sure node0 knows about the latest MaxShard.
maxShards0, err := client0.MaxShardByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices0["i"] != 2 {
t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"])
if maxShards0["i"] != 2 {
t.Fatalf("unexpected maxShard on node0: %d", maxShards0["i"])
}
// Make sure node1 knows about the latest MaxSlice.
maxSlices1, err := client1.MaxSliceByIndex(context.Background())
// Make sure node1 knows about the latest MaxShard.
maxShards1, err := client1.MaxShardByIndex(context.Background())
if err != nil {
t.Fatal(err)
}
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
if maxShards1["i"] != 2 {
t.Fatalf("unexpected maxShard on node1: %d", maxShards1["i"])
}
}
// Ensure that an empty node comes up in a NORMAL state.
func TestClusterResize_EmptyNode(t *testing.T) {
m0 := test.MustRunMain()
m0 := test.MustRunCommand()
defer m0.Close()
if m0.API.State() != pilosa.ClusterStateNormal {
@ -126,7 +126,7 @@ func TestClusterResize_EmptyNode(t *testing.T) {
// Ensure that a cluster of empty nodes comes up in a NORMAL state.
func TestClusterResize_EmptyNodes(t *testing.T) {
clus := test.MustRunMainWithCluster(t, 2)
clus := test.MustRunCluster(t, 2)
defer clus[0].Close()
defer clus[1].Close()
@ -140,7 +140,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) {
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNode(t *testing.T) {
t.Run("NoData", func(t *testing.T) {
clus := test.MustRunMainWithCluster(t, 2)
clus := test.MustRunCluster(t, 2)
if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
@ -150,7 +150,7 @@ func TestClusterResize_AddNode(t *testing.T) {
})
t.Run("WithIndex", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -166,7 +166,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -181,9 +181,9 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
})
t.Run("ContinuousSlices", func(t *testing.T) {
t.Run("ContinuousShards", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -207,7 +207,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -222,9 +222,9 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
})
t.Run("SkippedSlice", func(t *testing.T) {
t.Run("SkippedShard", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -239,7 +239,7 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatal(err)
}
// Write data on first node. Note that no data is placed on slice 1.
// Write data on first node. Note that no data is placed on shard 1.
if _, err := m0.Query("i", "", `
Set(1, f=1)
Set(2400000, f=1)
@ -248,7 +248,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -269,7 +269,7 @@ func TestClusterResize_AddNode(t *testing.T) {
func TestCluster_GossipMembership(t *testing.T) {
t.Run("Node0Down", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -277,7 +277,7 @@ func TestCluster_GossipMembership(t *testing.T) {
var eg errgroup.Group
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
defer m1.Close()
eg.Go(func() error {
m1.Config.Gossip.Port = "0"
@ -291,7 +291,7 @@ func TestCluster_GossipMembership(t *testing.T) {
})
// Configure node1
m2 := test.NewMainWithCluster(false)
m2 := test.NewCommandNode(false)
defer m2.Close()
eg.Go(func() error {
m2.Config.Gossip.Port = "0"
@ -324,7 +324,7 @@ func TestCluster_GossipMembership(t *testing.T) {
}
func TestClusterResize_RemoveNode(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3)
cluster := test.MustRunCluster(t, 3)
m0 := cluster[0]
m1 := cluster[1]
@ -390,7 +390,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setColumns := ""
for i := 0; i < 20; i++ {
setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.ShardWidth)
}
if _, err := m0.Query("i", "", setColumns); err != nil {
@ -410,7 +410,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// checkClusterState polls a given cluster for its state until it
// receives a matching state. It polls up to n times before returning.
func checkClusterState(m *test.Main, state string, n int) bool {
func checkClusterState(m *test.Command, state string, n int) bool {
for i := 0; i < n; i++ {
if m.API.State() == state {
return true

54
server/handler_test.go
View file

@ -37,7 +37,7 @@ import (
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_Endpoints(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -55,7 +55,7 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
} else if body := w.Body.String(); body != fmt.Sprintf("{\"shardWidth\":%d}\n", pilosa.ShardWidth) {
t.Fatalf("unexpected body: %s", body)
}
})
@ -112,19 +112,19 @@ func TestHandler_Endpoints(t *testing.T) {
// TODO need to test aborting a cluster resize job. this may not be the right place
})
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.ShardWidth)+4)
hldr.SetBit("i0", "f0", 31, 1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+8)
t.Run("Max Slice", func(t *testing.T) {
t.Run("Max Shard", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/shards/max", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
@ -132,9 +132,9 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
t.Run("Slices args", func(t *testing.T) {
t.Run("Shards args", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
@ -142,11 +142,11 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
t.Run("Slices args protobuf", func(t *testing.T) {
t.Run("Shards args protobuf", func(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
Query: "Count(Row(f0=30))",
Slices: []uint64{0, 1},
Shards: []uint64{0, 1},
})
if err != nil {
t.Fatal(err)
@ -169,17 +169,17 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Query args error", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=a,b", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
} else if body := w.Body.String(); body != `{"error":"invalid shard argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query params err", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1&db=sample", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
@ -217,9 +217,9 @@ func TestHandler_Endpoints(t *testing.T) {
})
f0 := i0.Field("f0")
if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil {
if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil {
} else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
} else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil {
t.Fatal(err)
@ -249,7 +249,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
@ -285,7 +285,7 @@ func TestHandler_Endpoints(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
@ -301,7 +301,7 @@ func TestHandler_Endpoints(t *testing.T) {
if a := resp.ColumnAttrSets; len(a) != 2 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != pilosa.SliceWidth+1 {
} else if a[0].ID != pilosa.ShardWidth+1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
@ -376,7 +376,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Err Parse", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?shards=0,1", strings.NewReader("bad_fn(")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" {
@ -507,7 +507,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Fragment Nodes", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil)
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&shard=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
@ -520,7 +520,7 @@ func TestHandler_Endpoints(t *testing.T) {
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&shard=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
@ -528,7 +528,7 @@ func TestHandler_Endpoints(t *testing.T) {
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?shard=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
@ -566,7 +566,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
clus := test.MustRunMainWithCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
w = httptest.NewRecorder()
h := clus[0].Handler.(*http.Handler).Handler
h.ServeHTTP(w, req)

8
server/server_test.go
View file

@ -40,7 +40,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
if err := quick.Check(func(cmds []SetCommand) bool {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create client.
@ -116,7 +116,7 @@ func TestMain_Set_Quick(t *testing.T) {
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create fields.
@ -193,7 +193,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create fields.
@ -264,7 +264,7 @@ func tempMkdir(t *testing.T) string {
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)
cluster := test.MustRunCluster(t, clusterSize)
// Create the schema.
client0 := cluster[0].Client()

2
server_test.go
View file

@ -28,7 +28,7 @@ import (
// pilosa.Server was not having its remoteClient field set by an option and so
// it was using a nil client in monitorAntiEntropy.
func TestMonitorAntiEntropy(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)})
cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)})
client := cluster[1].Client()
err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{})
if err != nil {

22
stats_test.go
View file

@ -39,43 +39,43 @@ func TestMultiStatClient_Expvar(t *testing.T) {
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
hldr.SetBit("d", "f", 0, ShardWidth)
hldr.SetBit("d", "f", 0, ShardWidth+2)
hldr.ClearBit("d", "f", 0, 1)
if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"})
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Gauge creates a unique key, subsequent Gauge calls will overwrite
hldr.Stats.Gauge("g", 5, 1.0)
hldr.Stats.Gauge("g", 8, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Set creates a unique key, subsequent sets will overwrite
hldr.Stats.Set("s", "4", 1.0)
hldr.Stats.Set("s", "7", 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Record timing duration and a uniquely Set key/value
dur, _ := time.ParseDuration("123us")
hldr.Stats.Timing("tt", dur, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Expvar histogram is implemented as a gauge
hldr.Stats.Histogram("hh", 3, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
@ -91,8 +91,8 @@ func TestStatsCount_TopN(t *testing.T) {
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
hldr.SetBit("d", "f", 0, ShardWidth)
hldr.SetBit("d", "f", 0, ShardWidth+2)
// Execute query.
called := false
@ -209,7 +209,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
}
func TestStatsCount_APICalls(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}

4
test/fragment.go
View file

@ -18,8 +18,8 @@ import (
"github.com/pilosa/pilosa"
)
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// ShardWidth is a helper reference to use when testing.
const ShardWidth = pilosa.ShardWidth
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {

31
test/handler.go
View file

@ -36,41 +36,16 @@ type Handler struct {
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler(opts ...http.HandlerOption) (*Handler, error) {
handler, err := http.NewHandler(opts...)
if err != nil {
return nil, err
}
h := &Handler{
Handler: handler,
}
// Handler test messages can no-op.
h.API.Broadcaster = pilosa.NopBroadcaster
return h, nil
}
// MustNewHandler returns a new instance of Handler.
func MustNewHandler(opts ...http.HandlerOption) *Handler {
h, err := NewHandler(opts...)
if err != nil {
panic(err)
}
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
ExecuteFn func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, slices, opt)
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, shards, opt)
}
// Server represents a test wrapper for httptest.Server.

6
test/holder.go
View file

@ -90,7 +90,7 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, shard uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
@ -100,7 +100,7 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string,
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
panic(err)
}
@ -152,7 +152,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
if err != nil {
panic(err)
}
f.ClearBit(rowID, columnID, nil)
f.ClearBit(rowID, columnID)
}
// MustSetBits sets columns on a row. Panic on error.

168
test/pilosa.go
View file

@ -32,8 +32,8 @@ import (
)
////////////////////////////////////////////////////////////////////////////////////
// Main represents a test wrapper for server.Command.
type Main struct {
// Command represents a test wrapper for server.Command.
type Command struct {
*server.Command
commandOptions []server.CommandOption
@ -57,21 +57,14 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
}
}
// GossipAddress returns the address on which gossip is listening after a Main
// has been setup. Useful to pass as a seed to other nodes when creating and
// testing clusters.
func (m *Main) GossipAddress() string {
return m.GossipTransport().URI.String()
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain(opts ...server.CommandOption) *Main {
// NewCommand returns a new instance of Main with a temporary data directory and random port.
func NewCommand(opts ...server.CommandOption) *Command {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
@ -92,58 +85,17 @@ func NewMain(opts ...server.CommandOption) *Main {
return m
}
// NewMainWithCluster returns a new instance of Main with clustering enabled.
func NewMainWithCluster(isCoordinator bool, opts ...server.CommandOption) *Main {
m := NewMain(opts...)
// NewCommandNode returns a new instance of Command with clustering enabled.
func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
m := NewCommand(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
}
// MustRunMainWithCluster ruturns a running array of *Main where
// all nodes are joined via memberlist (i.e. clustering enabled).
func MustRunMainWithCluster(t *testing.T, size int, opts ...[]server.CommandOption) []*Main {
ma, err := runMainWithCluster(size, opts...)
if err != nil {
t.Fatalf("new main array with cluster: %v", err)
}
return ma
}
// runMainWithCluster runs an array of *Main where all nodes are
// joined via memberlist (i.e. clustering enabled).
func runMainWithCluster(size int, opts ...[]server.CommandOption) ([]*Main, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
}
mains := make([]*Main, size)
var gossipSeeds = make([]string, size)
for i := 0; i < size; i++ {
var commandOpts []server.CommandOption
if len(opts) > 0 {
commandOpts = opts[i%len(opts)]
}
m := NewMainWithCluster(i == 0, commandOpts...)
m.Config.Gossip.Port = "0"
m.Config.Gossip.Seeds = gossipSeeds[:i]
if err := m.Start(); err != nil {
return nil, errors.Wrapf(err, "Starting server %d", i)
}
gossipSeeds[i] = m.GossipTransport().URI.String()
mains[i] = m
}
return mains, nil
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
// MustRunCommand returns a new, running Main. Panic on error.
func MustRunCommand() *Command {
m := NewCommand()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Start(); err != nil {
panic(err)
@ -151,14 +103,21 @@ func MustRunMain() *Main {
return m
}
// GossipAddress returns the address on which gossip is listening after a Main
// has been setup. Useful to pass as a seed to other nodes when creating and
// testing clusters.
func (m *Command) GossipAddress() string {
return m.GossipTransport().URI.String()
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
func (m *Command) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
func (m *Command) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
@ -180,10 +139,10 @@ func (m *Main) Reopen() error {
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return m.Server.URI.String() }
func (m *Command) URL() string { return m.Server.URI.String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *http.InternalClient {
func (m *Command) Client() *http.InternalClient {
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
panic(err)
@ -192,7 +151,7 @@ func (m *Main) Client() *http.InternalClient {
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
func (m *Command) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != gohttp.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
@ -200,7 +159,7 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) {
return resp.Body, nil
}
func (m *Main) RecalculateCaches() error {
func (m *Command) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
if resp.StatusCode != 204 {
return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
@ -208,6 +167,85 @@ func (m *Main) RecalculateCaches() error {
return nil
}
// Cluster represents a Pilosa cluster (multiple Command instances)
type Cluster []*Command
// Start runs a Cluster
func (c Cluster) Start() error {
var gossipSeeds = make([]string, len(c))
for i, cc := range c {
cc.Config.Gossip.Port = "0"
cc.Config.Gossip.Seeds = gossipSeeds[:i]
if err := cc.Start(); err != nil {
return errors.Wrapf(err, "starting server %d", i)
}
gossipSeeds[i] = cc.GossipAddress()
}
return nil
}
// Stop stops a Cluster
func (c Cluster) Close() error {
for i, cc := range c {
if err := cc.Close(); err != nil {
return errors.Wrapf(err, "stopping server %d", i)
}
}
return nil
}
// MustNewCluster creates a new cluster
func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster {
c, err := newCluster(size, opts...)
if err != nil {
t.Fatalf("new cluster: %v", err)
}
return c
}
// newCluster creates a new cluster
func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
}
cluster := make(Cluster, size)
for i := 0; i < size; i++ {
var commandOpts []server.CommandOption
if len(opts) > 0 {
commandOpts = opts[i%len(opts)]
}
m := NewCommandNode(i == 0, commandOpts...)
cluster[i] = m
}
return cluster, nil
}
// runCluster creates and starts a new cluster
func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
cluster, err := newCluster(size, opts...)
if err != nil {
return nil, errors.Wrap(err, "new cluster")
}
if err = cluster.Start(); err != nil {
return nil, errors.Wrap(err, "starting cluster")
}
return cluster, nil
}
// MustRunCluster creates and starts a new cluster
func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster {
c, err := runCluster(size, opts...)
if err != nil {
t.Fatalf("run cluster: %v", err)
}
return c
}
////////////////////////////////////////////////////////////////////////////////////
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.

4
test/pilosa_test.go
View file

@ -27,7 +27,7 @@ import (
func TestNewCluster(t *testing.T) {
numNodes := 3
cluster := test.MustRunMainWithCluster(t, numNodes)
cluster := test.MustRunCluster(t, numNodes)
coordinator := getCoordinator(cluster[0])
for i := 1; i < numNodes; i++ {
@ -78,7 +78,7 @@ func TestNewCluster(t *testing.T) {
}
}
func getCoordinator(m *test.Main) string {
func getCoordinator(m *test.Command) string {
hosts := m.API.Hosts(context.Background())
for _, host := range hosts {
if host.IsCoordinator {

4
translate.go
View file

@ -306,11 +306,13 @@ func (s *TranslateFile) replicate(ctx context.Context) error {
} else if err != nil {
return err
}
s.mu.Lock()
// Write to local store.
if err := s.appendEntry(&entry); err != nil {
s.mu.Unlock()
return err
}
s.mu.Unlock()
}
}

14
utils_internal_test.go
View file

@ -119,9 +119,9 @@ func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) erro
func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / SliceWidth
nodes := c0.sliceNodes(index, slice)
c0 := t.Clusters[0] // use the first node's cluster to determine shard location.
shard := colID / ShardWidth
nodes := c0.shardNodes(index, shard)
for _, node := range nodes {
c := t.clusterByID(node.ID)
@ -355,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI)
// basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI)
instrNode := DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
@ -368,14 +368,14 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
srcNode := DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice)
destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice)
srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard)
destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.holder.Field(src.Index, src.Field)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
destFragment, err = v.CreateFragmentIfNotExists(src.Shard)
if err != nil {
return err
}

109
view.go
View file

@ -49,17 +49,16 @@ type View struct {
cacheSize uint32
// Fragments by slice.
// Fragments by shard.
cacheType string // passed in by field
fragments map[uint64]*Fragment
// maxSlice maintains this view's max slice in order to
// prevent sending multiple `CreateSliceMessage` messages
maxSlice uint64
broadcaster Broadcaster
stats StatsClient
// maxShard maintains this view's max shard in order to
// prevent sending multiple `CreateShardMessage` messages
maxShard uint64
broadcaster Broadcaster
stats StatsClient
RowAttrStore AttrStore
Logger Logger
}
@ -132,17 +131,17 @@ func (v *View) openFragments() error {
}
// Parse filename into integer.
slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
continue
}
frag := v.newFragment(v.fragmentPath(slice), slice)
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err)
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
}
frag.RowAttrStore = v.RowAttrStore
v.fragments[frag.slice] = frag
v.fragments[frag.shard] = frag
}
return nil
@ -164,15 +163,15 @@ func (v *View) close() error {
return nil
}
// calculateMaxSlice returns the max slice in the view.
func (v *View) calculateMaxSlice() uint64 {
// calculateMaxShard returns the max shard in the view.
func (v *View) calculateMaxShard() uint64 {
v.mu.RLock()
defer v.mu.RUnlock()
var max uint64
for slice := range v.fragments {
if slice > max {
max = slice
for shard := range v.fragments {
if shard > max {
max = shard
}
}
@ -180,18 +179,18 @@ func (v *View) calculateMaxSlice() uint64 {
}
// fragmentPath returns the path to a fragment in the view.
func (v *View) fragmentPath(slice uint64) string {
return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10))
func (v *View) fragmentPath(shard uint64) string {
return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10))
}
// Fragment returns a fragment in the view by slice.
func (v *View) Fragment(slice uint64) *Fragment {
// Fragment returns a fragment in the view by shard.
func (v *View) Fragment(shard uint64) *Fragment {
v.mu.RLock()
defer v.mu.RUnlock()
return v.fragment(slice)
return v.fragment(shard)
}
func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] }
func (v *View) fragment(shard uint64) *Fragment { return v.fragments[shard] }
// allFragments returns a list of all fragments in the view.
func (v *View) allFragments() []*Fragment {
@ -212,64 +211,64 @@ func (v *View) recalculateCaches() {
}
}
// CreateFragmentIfNotExists returns a fragment in the view by slice.
func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
// CreateFragmentIfNotExists returns a fragment in the view by shard.
func (v *View) CreateFragmentIfNotExists(shard uint64) (*Fragment, error) {
v.mu.Lock()
defer v.mu.Unlock()
return v.createFragmentIfNotExists(slice)
return v.createFragmentIfNotExists(shard)
}
func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
func (v *View) createFragmentIfNotExists(shard uint64) (*Fragment, error) {
// Find fragment in cache first.
if frag := v.fragments[slice]; frag != nil {
if frag := v.fragments[shard]; frag != nil {
return frag, nil
}
// Initialize and open fragment.
frag := v.newFragment(v.fragmentPath(slice), slice)
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return nil, errors.Wrap(err, "opening fragment")
}
frag.RowAttrStore = v.RowAttrStore
// Broadcast a message that a new max slice was just created.
if slice > v.maxSlice {
v.maxSlice = slice
// Broadcast a message that a new max shard was just created.
if shard > v.maxShard {
v.maxShard = shard
// Send the create slice message to all nodes.
// Send the create shard message to all nodes.
err := v.broadcaster.SendSync(
&internal.CreateSliceMessage{
&internal.CreateShardMessage{
Index: v.index,
Slice: slice,
Shard: shard,
})
if err != nil {
return nil, errors.Wrap(err, "sending createslice message")
return nil, errors.Wrap(err, "sending createshard message")
}
}
// Save to lookup.
v.fragments[slice] = frag
v.fragments[shard] = frag
return frag, nil
}
func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.index, v.field, v.name, slice)
func (v *View) newFragment(path string, shard uint64) *Fragment {
frag := NewFragment(path, v.index, v.field, v.name, shard)
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
frag.Logger = v.Logger
frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
return frag
}
// deleteFragment removes the fragment from the view.
func (v *View) deleteFragment(slice uint64) error {
func (v *View) deleteFragment(shard uint64) error {
fragment := v.fragments[slice]
fragment := v.fragments[shard]
if fragment == nil {
return ErrFragmentNotFound
}
v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice)
v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
@ -283,15 +282,15 @@ func (v *View) deleteFragment(slice uint64) error {
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
v.Logger.Printf("no cache file to delete for slice %d", slice)
v.Logger.Printf("no cache file to delete for shard %d", shard)
}
delete(v.fragments, slice)
delete(v.fragments, shard)
return nil
}
// row returns a row for a slice of the view.
// row returns a row for a shard of the view.
func (v *View) row(rowID uint64) *Row {
row := NewRow()
for _, frag := range v.allFragments() {
@ -307,8 +306,8 @@ func (v *View) row(rowID uint64) *Row {
// setBit sets a bit within the view.
func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}
@ -317,18 +316,18 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) {
// clearBit clears a bit within the view.
func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
shard := columnID / ShardWidth
frag, found := v.fragments[shard]
if !found {
return false, nil
}
return frag.clearBit(rowID, columnID)
}
// value uses a column of bits to read a multi-bit value.
func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return value, exists, err
}
@ -337,8 +336,8 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool,
// setValue uses a column of bits to set a multi-bit value.
func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
shard := columnID / ShardWidth
frag, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
return changed, err
}

12
view_internal_test.go
View file

@ -39,27 +39,27 @@ func TestView_DeleteFragment(t *testing.T) {
v := mustOpenView("i", "f", "v")
defer v.close()
slice := uint64(9)
shard := uint64(9)
// Create fragment.
fragment, err := v.CreateFragmentIfNotExists(slice)
fragment, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
t.Fatal(err)
} else if fragment == nil {
t.Fatal("expected fragment")
}
err = v.deleteFragment(slice)
err = v.deleteFragment(shard)
if err != nil {
t.Fatal(err)
}
if v.Fragment(slice) != nil {
if v.Fragment(shard) != nil {
t.Fatal("fragment still exists in view")
}
// Recreate fragment with same slice, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(slice)
// Recreate fragment with same shard, verify that the old fragment was not reused.
fragment2, err := v.CreateFragmentIfNotExists(shard)
if err != nil {
t.Fatal(err)
} else if fragment == fragment2 {