From cabfa3c456997ea2f6d78cec9840d5b8dd72e085 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 12 Nov 2019 12:01:58 -0600 Subject: [PATCH 01/27] add a test for pilosa/#2084 --- field_internal_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/field_internal_test.go b/field_internal_test.go index 41cb5832b..f400ad3e9 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -529,3 +529,50 @@ func TestField_ApplyOptions(t *testing.T) { } } } + +// Ensure that importValue handles requiredDepth correctly. +// This test sets the same column value to 1, then 8, then 1. +// A previous bug was incorrectly determining bitDepth based +// on the values in the import, and not taking existing values +// into consideration. This would cause an import of 1/8/1 +// to result in a value of 9 instead of 1. +func TestBSIGroup_importValue(t *testing.T) { + f := MustOpenField(OptFieldTypeInt(-100, 200)) + + options := &ImportOptions{} + for i, tt := range []struct { + columnIDs []uint64 + values []int64 + checkVal int64 + expCols []uint64 + }{ + { + []uint64{100}, + []int64{1}, + 1, + []uint64{100}, + }, + { + []uint64{100}, + []int64{8}, + 8, + []uint64{100}, + }, + { + []uint64{100}, + []int64{1}, + 1, + []uint64{100}, + }, + } { + if err := f.importValue(tt.columnIDs, tt.values, options); err != nil { + t.Fatalf("test %d, importing values: %s", i, err.Error()) + } + + if row, err := f.Range(f.name, pql.EQ, tt.checkVal); err != nil { + t.Fatalf("test %d, getting range: %s", i, err.Error()) + } else if !reflect.DeepEqual(row.Columns(), tt.expCols) { + t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns()) + } + } +} From 7215bfd16c0a1a3f08ba00a7b2e3beee703082cf Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 2 Dec 2019 14:14:33 -0700 Subject: [PATCH 02/27] Implement translator store sharding --- api.go | 30 +- boltdb/translate.go | 17 +- client.go | 19 +- cluster.go | 532 ++++++- cluster_internal_test.go | 2 +- encoding/proto/proto.go | 30 +- executor.go | 187 ++- executor_internal_test.go | 18 +- field.go | 26 +- handler.go | 14 + holder.go | 257 +-- http/client.go | 100 ++ http/handler.go | 2 +- index.go | 33 +- internal/private.pb.go | 2846 ++++++++++++++++++++------------- internal/public.pb.go | 3191 ++++++++++++++++++++++++------------- internal/public.proto | 10 + mock/translator.go | 5 + pql/ast.go | 28 + server.go | 4 +- server/server.go | 1 - translate.go | 65 +- translator_test.go | 4 +- 23 files changed, 4717 insertions(+), 2704 deletions(-) diff --git a/api.go b/api.go index af468914f..03c3213e8 100644 --- a/api.go +++ b/api.go @@ -545,7 +545,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin var err error if field.keys() { - if rowStr, err = field.translateStore.TranslateID(rowID); err != nil { + if rowStr, err = api.cluster.translateFieldID(indexName, fieldName, rowID); err != nil { return errors.Wrap(err, "translating row") } } else { @@ -553,7 +553,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } if index.Keys() { - if colStr, err = index.translateStore.TranslateID(columnID); err != nil { + if colStr, err = api.cluster.translateIndexPartitionID(ctx, indexName, api.cluster.idPartition(indexName, columnID), columnID); err != nil { return errors.Wrap(err, "translating column") } } else { @@ -963,7 +963,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } - if req.RowIDs, err = field.translateStore.TranslateKeys(req.RowKeys); err != nil { + if req.RowIDs, err = api.cluster.translateFieldKeys(req.Index, req.Field, req.RowKeys); err != nil { return errors.Wrap(err, "translating rows") } } @@ -973,7 +973,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } - if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil { + if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil { return errors.Wrap(err, "translating columns") } } @@ -1078,7 +1078,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } - if req.ColumnIDs, err = index.translateStore.TranslateKeys(req.ColumnKeys); err != nil { + if req.ColumnIDs, err = api.cluster.translateIndexKeys(ctx, req.Index, req.ColumnKeys); err != nil { return errors.Wrap(err, "translating columns") } req.Shard = math.MaxUint64 @@ -1371,11 +1371,11 @@ func (api *API) Info() serverInfo { func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader") defer span.Finish() - return api.holder.TranslateEntryReader(ctx, offsets) + return api.cluster.translateEntryReader(ctx, offsets) } // TranslateKeys handles a TranslateKeyRequest. -func (api *API) TranslateKeys(r io.Reader) ([]byte, error) { +func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) { var req TranslateKeysRequest if buf, err := ioutil.ReadAll(r); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error")) @@ -1384,13 +1384,15 @@ func (api *API) TranslateKeys(r io.Reader) ([]byte, error) { } // Lookup store for either index or field and translate keys. - store, err := api.holder.TranslateStore(req.Index, req.Field) - if err != nil { - return nil, err - } - ids, err := store.TranslateKeys(req.Keys) - if err != nil { - return nil, err + var ids []uint64 + if req.Field == "" { + if ids, err = api.cluster.translateIndexKeys(ctx, req.Index, req.Keys); err != nil { + return nil, err + } + } else { + if ids, err = api.cluster.translateFieldKeys(req.Index, req.Field, req.Keys); err != nil { + return nil, err + } } // Encode response. diff --git a/boltdb/translate.go b/boltdb/translate.go index bca1e5f92..dcfccc5b5 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -32,8 +32,8 @@ var ( ) // OpenTranslateStore opens and initializes a boltdb translation store. -func OpenTranslateStore(path, index, field string) (pilosa.TranslateStore, error) { - s := NewTranslateStore(index, field) +func OpenTranslateStore(path, index, field string, partitionID int) (pilosa.TranslateStore, error) { + s := NewTranslateStore(index, field, partitionID) s.Path = path if err := s.Open(); err != nil { return nil, err @@ -49,8 +49,9 @@ type TranslateStore struct { mu sync.RWMutex db *bolt.DB - index string - field string + index string + field string + partitionID int once sync.Once closing chan struct{} @@ -63,10 +64,11 @@ type TranslateStore struct { } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(index, field string) *TranslateStore { +func NewTranslateStore(index, field string, partitionID int) *TranslateStore { return &TranslateStore{ index: index, field: field, + partitionID: partitionID, closing: make(chan struct{}), writeNotify: make(chan struct{}), } @@ -108,6 +110,11 @@ func (s *TranslateStore) Close() (err error) { return nil } +// PartitionID returns the partition id the store was initialized with. +func (s *TranslateStore) PartitionID() int { + return s.partitionID +} + // ReadOnly returns true if the store is in read-only mode. func (s *TranslateStore) ReadOnly() bool { s.mu.RLock() diff --git a/client.go b/client.go index 03203b8a9..ffeb19b07 100644 --- a/client.go +++ b/client.go @@ -44,6 +44,8 @@ type FieldValue struct { // While I understand that putting the entire Client behind an interface might require this many methods, // I don't want to let it go unquestioned. type InternalClient interface { + InternalQueryClient + MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error @@ -51,7 +53,6 @@ type InternalClient interface { FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Nodes(ctx context.Context) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error @@ -78,6 +79,8 @@ type InternalClient interface { // InternalQueryClient is the internal interface for querying a node. type InternalQueryClient interface { QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) + TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) + TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error) } type nopInternalQueryClient struct{} @@ -86,6 +89,14 @@ func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index return nil, nil } +func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) { + return nil, nil +} + +func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) { + return nil, nil +} + func newNopInternalQueryClient() *nopInternalQueryClient { return &nopInternalQueryClient{} } @@ -125,6 +136,12 @@ func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { return nil, nil } +func (n nopInternalClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string) ([]uint64, error) { + return nil, nil +} +func (n nopInternalClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) { + return nil, nil +} func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error { return nil } diff --git a/cluster.go b/cluster.go index e723de58e..92516fb7c 100644 --- a/cluster.go +++ b/cluster.go @@ -215,6 +215,10 @@ type cluster struct { // nolint: maligned holder *Holder broadcaster broadcaster + // translation stores + indexTranslateStoreMap map[string]map[int]TranslateStore // store by index+partition + fieldTranslateStoreMap map[string]map[string]TranslateStore // store by index+field + joiningLeavingNodes chan nodeAction // joining is held open until this node @@ -235,6 +239,10 @@ type cluster struct { // nolint: maligned logger logger.Logger InternalClient InternalClient + + // Instantiates new translation stores + OpenTranslateStore OpenTranslateStoreFunc + OpenTranslateReader OpenTranslateReaderFunc } // newCluster returns a new instance of Cluster with defaults. @@ -249,8 +257,13 @@ func newCluster() *cluster { closing: make(chan struct{}), joining: make(chan struct{}), + indexTranslateStoreMap: make(map[string]map[int]TranslateStore), + fieldTranslateStoreMap: make(map[string]map[string]TranslateStore), + InternalClient: newNopInternalClient(), + OpenTranslateStore: OpenInMemTranslateStore, + logger: logger.NopLogger, } } @@ -390,14 +403,6 @@ func (c *cluster) addNode(node *Node) error { return nil } - // If the cluster membership has changed, reset the primary for - // translate store replication. - if c.holder != nil { - if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil { - return err - } - } - // add to topology if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") @@ -417,14 +422,6 @@ func (c *cluster) removeNode(nodeID string) error { // remove from cluster c.removeNodeBasicSorted(nodeID) - // If the cluster membership has changed, reset the primary for - // translate store replication. - if c.holder != nil { - if err := c.holder.setPrimaryTranslateStore(c.unprotectedPrimaryReplicaNode()); err != nil { - return err - } - } - // remove from topology if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") @@ -867,8 +864,8 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour return m, nil } -// partition returns the partition that a shard belongs to. -func (c *cluster) partition(index string, shard uint64) int { +// shardPartition returns the partition that a shard belongs to. +func (c *cluster) shardPartition(index string, shard uint64) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], shard) @@ -879,6 +876,20 @@ func (c *cluster) partition(index string, shard uint64) int { return int(h.Sum64() % uint64(c.partitionN)) } +// keyPartition returns the partition that a shard belongs to. +func (c *cluster) keyPartition(index, key string) int { + // Hash the bytes and mod by partition count. + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write([]byte(key)) + return int(h.Sum64() % uint64(c.partitionN)) +} + +// idPartition returns the partition that an id belongs to. +func (c *cluster) idPartition(index string, id uint64) int { + return c.shardPartition(index, id/ShardWidth) +} + // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. func (c *cluster) ShardNodes(index string, shard uint64) []*Node { c.mu.RLock() @@ -888,7 +899,19 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node { // shardNodes returns a list of nodes that own a fragment. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { - return c.partitionNodes(c.partition(index, shard)) + return c.partitionNodes(c.shardPartition(index, shard)) +} + +// KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use. +func (c *cluster) KeyNodes(index, key string) []*Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.keyNodes(index, key) +} + +// keyNodes returns a list of nodes that own a key. unprotected +func (c *cluster) keyNodes(index, key string) []*Node { + return c.partitionNodes(c.keyPartition(index, key)) } // ownsShard returns true if a host owns a fragment. @@ -900,7 +923,6 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool { // partitionNodes returns a list of nodes that own a partition. unprotected. func (c *cluster) partitionNodes(partitionID int) []*Node { - // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. replicaN := c.ReplicaN @@ -922,11 +944,18 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { return nodes } +// ownsPartition returns true if a host owns a partition. +func (c *cluster) ownsPartition(nodeID string, partition int) bool { + c.mu.RLock() + defer c.mu.RUnlock() + return Nodes(c.partitionNodes(partition)).ContainsID(nodeID) +} + // containsShards is like OwnsShards, but it includes replicas. func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { var shards []uint64 availableShards.ForEach(func(i uint64) { - p := c.partition(index, i) + p := c.shardPartition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) for _, n := range nodes { @@ -982,6 +1011,10 @@ func (c *cluster) setup() error { if err != nil { return errors.Wrap(err, "adding local node") } + + if err := c.updateTranslateStores(); err != nil { + return err + } return nil } @@ -1991,6 +2024,11 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } + // Open appropriate translate stores. + if err := c.updateTranslateStores(); err != nil { + return err + } + c.unprotectedSetState(cs.State) c.markAsJoined() @@ -2047,6 +2085,458 @@ func (c *cluster) setStatic(hosts []string) error { return nil } +// updateTranslateStores starts stores for partitions & fields owned by this +// node and stops stores for ones this node does not own. +func (c *cluster) updateTranslateStores() error { + if err := c.closeUnownedTranslateStores(); err != nil { + return err + } else if err := c.openOwnedTranslateStores(); err != nil { + return err + } + return nil +} + +func (c *cluster) closeUnownedTranslateStores() error { + for indexName, m := range c.indexTranslateStoreMap { + idx := c.holder.Index(indexName) + + // Close unowned partition stores. + for partitionID, store := range m { + if idx != nil && c.ownsShard(c.Node.ID, indexName, uint64(partitionID)) { + continue + } + if err := store.Close(); err != nil { + return err + } + delete(m, partitionID) + } + if len(c.indexTranslateStoreMap) == 0 { + delete(c.indexTranslateStoreMap, indexName) + } + } + + // Close index partition stores. + for indexName, m := range c.fieldTranslateStoreMap { + idx := c.holder.Index(indexName) + + for fieldName, store := range m { + if idx != nil && idx.Field(fieldName) != nil { + continue + } + if err := store.Close(); err != nil { + return err + } + delete(m, fieldName) + } + if len(c.fieldTranslateStoreMap) == 0 { + delete(c.fieldTranslateStoreMap, indexName) + } + } + return nil +} + +// openOwnedTranslateStores ensures that all owned partition & field stores are open. +func (c *cluster) openOwnedTranslateStores() error { + // Open partition stores. + for _, index := range c.holder.Indexes() { + m := c.indexTranslateStoreMap[index.Name()] + if m == nil { + m = make(map[int]TranslateStore) + c.indexTranslateStoreMap[index.Name()] = m + } + + for partitionID := 0; partitionID < c.partitionN; partitionID++ { + if m[partitionID] != nil { + continue + } + store, err := c.OpenTranslateStore(index.TranslateStorePath(partitionID), index.Name(), "", partitionID) + if err != nil { + return err + } + m[partitionID] = store + } + } + + // Open field stores. + for _, index := range c.holder.Indexes() { + m := c.fieldTranslateStoreMap[index.Name()] + if m == nil { + m = make(map[string]TranslateStore) + c.fieldTranslateStoreMap[index.Name()] = m + } + + for _, field := range index.Fields() { + if m[field.Name()] != nil { + continue + } + store, err := c.OpenTranslateStore(field.TranslateStorePath(), index.Name(), field.Name(), 0) + if err != nil { + return err + } + m[field.Name()] = store + } + } + + return nil +} + +func (c *cluster) indexPartitionTranslateStore(index string, partitionID int) TranslateStore { + m := c.indexTranslateStoreMap[index] + if m == nil { + return nil + } + return m[partitionID] +} + +func (c *cluster) fieldTranslateStore(index, field string) TranslateStore { + m := c.fieldTranslateStoreMap[index] + if m == nil { + return nil + } + return m[field] +} + +func (c *cluster) translateIndexKeys(ctx context.Context, index string, keys []string) ([]uint64, error) { + keySet := make(map[string]struct{}) + for _, key := range keys { + keySet[key] = struct{}{} + } + + keyMap, err := c.translateIndexKeySet(ctx, index, keySet) + if err != nil { + return nil, err + } + + ids := make([]uint64, len(keys)) + for i := range keys { + ids[i] = keyMap[keys[i]] + } + return ids, nil +} + +func (c *cluster) translateIndexKeySet(ctx context.Context, index string, keySet map[string]struct{}) (map[string]uint64, error) { + keyMap := make(map[string]uint64) + + // Split keys by partition. + keysByPartition := make(map[int][]string, c.partitionN) + for key := range keySet { + partitionID := c.keyPartition(index, key) + keysByPartition[partitionID] = append(keysByPartition[partitionID], key) + } + + // Translate keys by partition. + var g errgroup.Group + var mu sync.Mutex + for partitionID := range keysByPartition { + keys := keysByPartition[partitionID] + + g.Go(func() (err error) { + var ids []uint64 + if c.ownsPartition(c.Node.ID, partitionID) { + if ids, err = c.translateIndexPartitionKeys(ctx, index, partitionID, keys); err != nil { + return err + } + } else { + nodes := c.partitionNodes(partitionID) + if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, index, "", keys); err != nil { + return err + } + } + + mu.Lock() + defer mu.Unlock() + for i := range keys { + keyMap[keys[i]] = ids[i] + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return keyMap, nil +} + +func (c *cluster) translateIndexIDSet(ctx context.Context, index string, idSet map[uint64]struct{}) (map[uint64]string, error) { + idMap := make(map[uint64]string) + + // Split ids by partition. + idsByPartition := make(map[int][]uint64, c.partitionN) + for id := range idSet { + partitionID := c.idPartition(index, id) + idsByPartition[partitionID] = append(idsByPartition[partitionID], id) + } + + // Translate ids by partition. + var g errgroup.Group + var mu sync.Mutex + for partitionID, ids := range idsByPartition { + g.Go(func() error { + nodes := c.partitionNodes(partitionID) + keys, err := c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, index, "", ids) + if err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + for i := range ids { + idMap[ids[i]] = keys[i] + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return idMap, nil +} + +func (c *cluster) translateIndexPartitionKeys(ctx context.Context, index string, partitionID int, keys []string) ([]uint64, error) { + s := c.indexPartitionTranslateStore(index, partitionID) + if s == nil { + return nil, ErrTranslateStoreNotFound + } + return s.TranslateKeys(keys) +} + +func (c *cluster) translateIndexPartitionIDs(ctx context.Context, index string, partitionID int, ids []uint64) ([]string, error) { + s := c.indexPartitionTranslateStore(index, partitionID) + if s == nil { + return nil, ErrTranslateStoreNotFound + } + return s.TranslateIDs(ids) +} + +func (c *cluster) translateIndexPartitionID(ctx context.Context, index string, partitionID int, id uint64) (string, error) { + s := c.indexPartitionTranslateStore(index, partitionID) + if s == nil { + return "", ErrTranslateStoreNotFound + } + return s.TranslateID(id) +} + +func (c *cluster) translateFieldKey(index, field string, key string) (uint64, error) { + s := c.fieldTranslateStore(index, field) + if s == nil { + return 0, ErrTranslateStoreNotFound + } + return s.TranslateKey(key) +} + +func (c *cluster) translateFieldKeys(index, field string, keys []string) ([]uint64, error) { + s := c.fieldTranslateStore(index, field) + if s == nil { + return nil, ErrTranslateStoreNotFound + } + return s.TranslateKeys(keys) +} + +func (c *cluster) translateFieldID(index, field string, id uint64) (string, error) { + s := c.fieldTranslateStore(index, field) + if s == nil { + return "", ErrTranslateStoreNotFound + } + return s.TranslateID(id) +} + +func (c *cluster) translateFieldIDs(index, field string, ids []uint64) ([]string, error) { + s := c.fieldTranslateStore(index, field) + if s == nil { + return nil, ErrTranslateStoreNotFound + } + return s.TranslateIDs(ids) +} + +// TranslateOffsetMap returns a map of offsets for all indexes & fields. +func (c *cluster) TranslateOffsetMap() (TranslateOffsetMap, error) { + m := make(TranslateOffsetMap) + + for index, partitionMap := range c.indexTranslateStoreMap { + for partitionID, store := range partitionMap { + id, err := store.MaxID() + if err != nil { + return nil, err + } + m.SetIndexPartitionOffset(index, partitionID, id+1) + } + } + + for index, fieldMap := range c.fieldTranslateStoreMap { + for field, store := range fieldMap { + id, err := store.MaxID() + if err != nil { + return nil, err + } + m.SetFieldOffset(index, field, id+1) + } + } + + return m, nil +} + +func (c *cluster) setTranslateStoreReadOnly(v bool) { + for _, partitionMap := range c.indexTranslateStoreMap { + for _, store := range partitionMap { + store.SetReadOnly(v) + } + } + + for _, fieldMap := range c.fieldTranslateStoreMap { + for _, store := range fieldMap { + store.SetReadOnly(v) + } + } +} + +// translateEntryReader returns a reader that merges all index & field reader +// that are specified in the offsets map. +func (c *cluster) translateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { + // Ensure all readers are cleaned up if any error. + var a []TranslateEntryReader + defer func() { + if err != nil { + for i := range a { + a[i].Close() + } + } + }() + + // Fetch all index partition readers. + for indexName, indexMap := range offsets { + for partitionID, offset := range indexMap.Partitions { + store := c.indexPartitionTranslateStore(indexName, partitionID) + if store == nil { + return nil, ErrTranslateStoreNotFound + } + + r, err := store.EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "index partition translate reader") + } + a = append(a, r) + } + } + + // Fetch all field readers. + for indexName, indexMap := range offsets { + for fieldName, offset := range indexMap.Fields { + store := c.fieldTranslateStore(indexName, fieldName) + if store == nil { + return nil, ErrTranslateStoreNotFound + } + + r, err := store.EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "field translate reader") + } + a = append(a, r) + } + } + + return NewMultiTranslateEntryReader(ctx, a), nil +} + +/* +// holderTranslateStoreReplicator manages the replication of translation store +// data from a primary store to the local replica. Continually tries to +// reconnect on disconnect. +type holderTranslateStoreReplicator struct { + ctx context.Context + cancel func() + wg sync.WaitGroup + + holder *Holder + nodeURL string + + logger logger.Logger +} + +func newHolderTranslateStoreReplicator(h *Holder, nodeURL string) *holderTranslateStoreReplicator { + r := &holderTranslateStoreReplicator{ + holder: h, + nodeURL: nodeURL, + logger: logger.NopLogger, + } + r.ctx, r.cancel = context.WithCancel(context.Background()) + return r +} + +// Open starts the background monitoring goroutine. +func (r *holderTranslateStoreReplicator) Open() error { + r.wg.Add(1) + go func() { defer r.wg.Done(); r.monitor() }() + return nil +} + +// Close stops the replicator. +func (r *holderTranslateStoreReplicator) Close() error { + r.cancel() + return nil +} + +// monitor runs in a background goroutine and continually tries to connect and +// stream translate changes from the primary store. +func (r *holderTranslateStoreReplicator) monitor() { + for { + select { + case <-r.ctx.Done(): + return + default: + if err := r.replicate(); err != nil { + r.logger.Printf("cannot replicate: nodeURL=%s err=%s", r.nodeURL, err) + } + time.Sleep(1 * time.Second) + } + } +} + +func (r *holderTranslateStoreReplicator) replicate() error { + // Determine the offsets of every index & field store. + offsets, err := r.holder.TranslateOffsetMap() + if err != nil { + return err + } else if len(offsets) == 0 { + return nil + } + + // Begin streaming from remote primary. + rd, err := r.holder.OpenTranslateReader(r.ctx, r.nodeURL, offsets) + if err != nil { + return err + } + defer rd.Close() + + for { + var entry TranslateEntry + if err := rd.ReadEntry(&entry); err != nil { + return err + } + + // Find appropriate store. + var store TranslateStore + if entry.Field == "" { + idx := r.holder.Index(entry.Index) + if idx == nil { + return ErrIndexNotFound + } + store = idx.TranslateStore() + } else { + f := r.holder.Field(entry.Index, entry.Field) + if f == nil { + return ErrFieldNotFound + } + store = f.TranslateStore() + } + + // Apply replication to store. + if err := store.ForceSet(entry.ID, entry.Key); err != nil { + return err + } + } +} +*/ + // ClusterStatus describes the status of the cluster including its // state and node topology. type ClusterStatus struct { diff --git a/cluster_internal_test.go b/cluster_internal_test.go index fe519ed04..97eb73fbd 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -352,7 +352,7 @@ func TestCluster_Partition(t *testing.T) { c := newCluster() c.partitionN = partitionN - partitionID := c.partition(index, shard) + partitionID := c.shardPartition(index, shard) if partitionID < 0 || partitionID >= partitionN { t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c1644ed58..d75b784e1 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -758,17 +758,31 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac return &internal.RecalculateCaches{} } +func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest { + return &internal.TranslateKeysRequest{ + Index: request.Index, + Field: request.Field, + Keys: request.Keys, + } +} + func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse { return &internal.TranslateKeysResponse{ IDs: response.IDs, } } -func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest { - return &internal.TranslateKeysRequest{ +func encodeTranslateIDsRequest(request *pilosa.TranslateIDsRequest) *internal.TranslateIDsRequest { + return &internal.TranslateIDsRequest{ Index: request.Index, Field: request.Field, - Keys: request.Keys, + IDs: request.IDs, + } +} + +func encodeTranslateIDsResponse(response *pilosa.TranslateIDsResponse) *internal.TranslateIDsResponse { + return &internal.TranslateIDsResponse{ + Keys: response.Keys, } } @@ -1102,6 +1116,16 @@ func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.T m.IDs = pb.IDs } +func decodeTranslateIDsRequest(pb *internal.TranslateIDsRequest, m *pilosa.TranslateIDsRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.IDs = pb.IDs +} + +func decodeTranslateIDsResponse(pb *internal.TranslateIDsResponse, m *pilosa.TranslateIDsResponse) { + m.Keys = pb.Keys +} + // QueryResult types. const ( queryResultTypeNil uint32 = iota diff --git a/executor.go b/executor.go index db53dad27..ddec4e18b 100644 --- a/executor.go +++ b/executor.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // defaultField is the field used if one is not specified. @@ -230,12 +231,18 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // Translate column attributes, if necessary. if idx.Keys() { + idSet := make(map[uint64]struct{}) for _, col := range columnAttrSets { - v, err := idx.translateStore.TranslateID(col.ID) - if err != nil { - return resp, err - } - col.Key, col.ID = v, 0 + idSet[col.ID] = struct{}{} + } + + idMap, err := e.Cluster.translateIndexIDSet(ctx, index, idSet) + if err != nil { + return resp, errors.Wrap(err, "translating id set") + } + + for _, col := range columnAttrSets { + col.Key, col.ID = idMap[col.ID], 0 } } @@ -3520,10 +3527,27 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } } -func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, calls []*pql.Call) error { +func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, calls []*pql.Call) (err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateCalls") defer span.Finish() + // TODO(BBJ): Handle cross-index boundaries. + keyMap := make(map[string]uint64) + if idx.Keys() { + // Collect all index keys. + keySet := make(map[string]struct{}) + for i := range calls { + if err := e.collectCallIndexKeys(index, idx, calls[i], keySet); err != nil { + return err + } + } + + if keyMap, err = e.Cluster.translateIndexKeySet(ctx, index, keySet); err != nil { + return err + } + } + + // Translate calls. for i := range calls { // Possibly change to another index for translation, if this // call crosses index boundaries. @@ -3538,51 +3562,57 @@ func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, return fmt.Errorf("unknown index %q specified in cross-index call", newIdxName) } } - if err := e.translateCall(newIdxName, newIdx, calls[i]); err != nil { + if err := e.translateCall(newIdxName, newIdx, calls[i], keyMap); err != nil { return err } } return nil } -func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { - var colKey, rowKey, fieldName string - switch c.Name { - case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow": - // Positional args in new PQL syntax require special handling here. - colKey = "_" + columnLabel - fieldName, _ = c.FieldArg() - rowKey = fieldName - case "SetRowAttrs": - // Positional args in new PQL syntax require special handling here. - rowKey = "_" + rowLabel - fieldName = callArgString(c, "_field") - case "Rows": - fieldName = callArgString(c, "_field") - rowKey = "previous" - colKey = "column" - case "GroupBy": - return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy") - case "IncludesColumn": - colKey = "column" - default: - colKey = "col" - fieldName = callArgString(c, "field") - rowKey = "row" +func (e *executor) collectCallIndexKeys(index string, idx *Index, c *pql.Call, keySet map[string]struct{}) error { + // Handle group by separately. + if c.Name == "GroupBy" { + for _, child := range c.Children { + if err := e.collectCallIndexKeys(index, idx, child, keySet); err != nil { + return errors.Wrapf(err, "translating %s", child) + } + } + + if filter, ok, err := c.CallArg("filter"); ok { + if err != nil { + return errors.Wrap(err, "getting filter call") + } + err = e.collectCallIndexKeys(index, idx, filter, keySet) + if err != nil { + return errors.Wrap(err, "translating filter call") + } + } + return nil + } + + colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) + if c.Args[colKey] != nil && !isString(c.Args[colKey]) { + return errors.New("column value must be a string when index 'keys' option enabled") + } else if value := callArgString(c, colKey); value != "" { + keySet[value] = struct{}{} + } + return nil +} + +func (e *executor) translateCall(index string, idx *Index, c *pql.Call, keyMap map[string]uint64) error { + if c.Name == "GroupBy" { + return errors.Wrap(e.translateGroupByCall(index, idx, c, keyMap), "translating GroupBy") } // Translate column key. + colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) if idx.Keys() { if c.Args[colKey] != nil && !isString(c.Args[colKey]) { if !isValidID(c.Args[colKey]) { return errors.Errorf("column value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[colKey]) } } else if value := callArgString(c, colKey); value != "" { - id, err := idx.translateStore.TranslateKey(value) - if err != nil { - return err - } - c.Args[colKey] = id + c.Args[colKey] = keyMap[value] } } else { if isString(c.Args[colKey]) { @@ -3626,7 +3656,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) } } else if value := callArgString(c, rowKey); value != "" { - id, err := field.translateStore.TranslateKey(value) + id, err := e.Cluster.translateFieldKey(index, fieldName, value) if err != nil { return err } @@ -3654,7 +3684,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { return fmt.Errorf("unknown index %q specified in cross-index call", newIdxName) } } - if err := e.translateCall(newIdxName, newIdx, child); err != nil { + if err := e.translateCall(newIdxName, newIdx, child, keyMap); err != nil { return err } } @@ -3662,13 +3692,13 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { return nil } -func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) error { +func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call, keyMap map[string]uint64) error { if c.Name != "GroupBy" { panic("translateGroupByCall called with '" + c.Name + "'") } for _, child := range c.Children { - if err := e.translateCall(index, idx, child); err != nil { + if err := e.translateCall(index, idx, child, keyMap); err != nil { return errors.Wrapf(err, "translating %s", child) } } @@ -3677,7 +3707,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e if err != nil { return errors.Wrap(err, "getting filter call") } - err = e.translateCall(index, idx, filter) + err = e.translateCall(index, idx, filter, keyMap) if err != nil { return errors.Wrap(err, "translating filter call") } @@ -3687,7 +3717,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e if err != nil { return errors.Wrap(err, "getting aggregate call") } - err = e.translateCall(index, idx, aggregate) + err = e.translateCall(index, idx, aggregate, keyMap) if err != nil { return errors.Wrap(err, "translating aggregate call") } @@ -3722,7 +3752,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") } - id, err := field.translateStore.TranslateKey(prevStr) + id, err := e.Cluster.translateFieldKey(index, field.Name(), prevStr) if err != nil { return errors.Wrapf(err, "translating row key '%s'", prevStr) } @@ -3741,8 +3771,49 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults") defer span.Finish() + idMap := make(map[uint64]string) + if idx.Keys() { + // Collect all index ids. + idSet := make(map[uint64]struct{}) + for i := range calls { + if err := e.collectResultIDs(index, idx, calls[i], results[i], idSet); err != nil { + return err + } + } + + // Split ids by partition. + idsByPartition := make(map[int][]uint64, e.Cluster.partitionN) + for id := range idSet { + partitionID := e.Cluster.shardPartition(index, id/ShardWidth) + idsByPartition[partitionID] = append(idsByPartition[partitionID], id) + } + + // Translate ids by partition. + var g errgroup.Group + var mu sync.Mutex + for partitionID, ids := range idsByPartition { + g.Go(func() error { + nodes := e.Cluster.partitionNodes(partitionID) + keys, err := e.client.TranslateIDsNode(ctx, &nodes[0].URI, index, "", ids) + if err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + for i := range ids { + idMap[ids[i]] = keys[i] + } + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + } + for i := range results { - results[i], err = e.translateResult(index, idx, calls[i], results[i]) + results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap) if err != nil { return err } @@ -3750,18 +3821,30 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde return nil } -func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { +func (e *executor) collectResultIDs(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]struct{}) error { + row, ok := result.(*Row) + if !ok { + return nil + } else if !idx.Keys() { + return nil + } + + for _, segment := range row.Segments() { + for _, col := range segment.Columns() { + idSet[col] = struct{}{} + } + } + return nil +} + +func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}, idSet map[uint64]string) (interface{}, error) { switch result := result.(type) { case *Row: if idx.Keys() { other := &Row{Attrs: result.Attrs} for _, segment := range result.Segments() { for _, col := range segment.Columns() { - key, err := idx.translateStore.TranslateID(col) - if err != nil { - return nil, err - } - other.Keys = append(other.Keys, key) + other.Keys = append(other.Keys, idSet[col]) } } return other, nil @@ -3825,7 +3908,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, ErrFieldNotFound } if field.keys() { - key, err := field.translateStore.TranslateID(g.RowID) + key, err := e.Cluster.translateFieldID(index, g.Field, g.RowID) if err != nil { return nil, errors.Wrap(err, "translating row ID in Group") } @@ -3856,7 +3939,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } else if field.keys() { other.Keys = make([]string, len(result)) for i, id := range result { - key, err := field.translateStore.TranslateID(id) + key, err := e.Cluster.translateFieldID(index, fieldName, id) if err != nil { return nil, errors.Wrap(err, "translating row ID") } diff --git a/executor_internal_test.go b/executor_internal_test.go index c74018a01..16588c1e2 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -25,8 +25,16 @@ import ( ) func TestExecutor_TranslateGroupByCall(t *testing.T) { + holder := NewHolder() + + cluster := newCluster() + cluster.holder = holder + cluster.Node = &Node{ID: "node1", URI: NewTestURIFromHostPort("node1", 0)} + cluster.addNode(cluster.Node) + e := &executor{ - Holder: NewHolder(), + Holder: holder, + Cluster: cluster, } e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") err := e.Holder.Open() @@ -46,12 +54,16 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("creating fields %v, %v, %v", erra, errb, errc) } + if err := cluster.updateTranslateStores(); err != nil { + t.Fatal(err) + } + query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`) if err != nil { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, c) + err = e.translateGroupByCall("i", idx, c, make(map[string]uint64)) if err != nil { t.Fatalf("translating call: %v", err) } @@ -115,7 +127,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, c) + err = e.translateGroupByCall("i", idx, c, make(map[string]uint64)) if err == nil { t.Fatalf("expected error, but translated call is '%s", c) } diff --git a/field.go b/field.go index 86c5204a8..842be406d 100644 --- a/field.go +++ b/field.go @@ -75,9 +75,6 @@ type Field struct { // Row attribute storage and cache rowAttrStore AttrStore - // Key/ID translation store. - translateStore TranslateStore - broadcaster broadcaster Stats stats.StatsClient @@ -92,8 +89,6 @@ type Field struct { logger logger.Logger snapshotQueue snapshotQueue - // Instantiates new translation store on open. - OpenTranslateStore OpenTranslateStoreFunc } // FieldOption is a functional option type for pilosa.fieldOptions. @@ -265,8 +260,6 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { remoteAvailableShards: roaring.NewBitmap(), logger: logger.NopLogger, - - OpenTranslateStore: OpenInMemTranslateStore, } return f, nil } @@ -280,12 +273,14 @@ func (f *Field) Index() string { return f.index } // Path returns the path the field was initialized with. func (f *Field) Path() string { return f.path } +// TranslateStorePath returns the translation database path for the field. +func (f *Field) TranslateStorePath() string { + return filepath.Join(f.path, "keys") +} + // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } -// TranslateStore returns the underlying translation store for the field. -func (f *Field) TranslateStore() TranslateStore { return f.translateStore } - // AvailableShards returns a bitmap of shards that contain data. func (f *Field) AvailableShards() *roaring.Bitmap { f.mu.RLock() @@ -473,11 +468,6 @@ func (f *Field) Open() error { return errors.Wrap(err, "opening attrstore") } - // Instantiate & open translation store. - if f.translateStore, err = f.OpenTranslateStore(filepath.Join(f.path, "keys"), f.index, f.name); err != nil { - return errors.Wrap(err, "opening translate store") - } - return nil }(); err != nil { f.Close() @@ -726,12 +716,6 @@ func (f *Field) Close() error { } f.viewMap = make(map[string]*view) - if f.translateStore != nil { - if err := f.translateStore.Close(); err != nil { - return err - } - } - return nil } diff --git a/handler.go b/handler.go index 19fcbac44..54444a735 100644 --- a/handler.go +++ b/handler.go @@ -212,3 +212,17 @@ type TranslateKeysRequest struct { type TranslateKeysResponse struct { IDs []uint64 } + +// TranslateIDsRequest describes the structure of a request +// for a batch of id translations. +type TranslateIDsRequest struct { + Index string + Field string + IDs []uint64 +} + +// TranslateIDsResponse is the structured response of a id +// translation request. +type TranslateIDsResponse struct { + Keys []string +} diff --git a/holder.go b/holder.go index 5045bff4d..2bc300ec1 100644 --- a/holder.go +++ b/holder.go @@ -78,12 +78,7 @@ type Holder struct { snapshotQueue snapshotQueue // Manages replication from the primary node. - primaryTranslateNode *Node - translateStoreReplicator *holderTranslateStoreReplicator - - // Instantiates new translation stores for indexes & fields. - OpenTranslateStore OpenTranslateStoreFunc // local store - OpenTranslateReader OpenTranslateReaderFunc // replication + primaryTranslateNode *Node } // lockedChan looks a little ridiculous admittedly, but exists for good reason. @@ -128,8 +123,6 @@ func NewHolder() *Holder { cacheFlushInterval: defaultCacheFlushInterval, Logger: logger.NopLogger, - - OpenTranslateStore: OpenInMemTranslateStore, } } @@ -232,13 +225,6 @@ func (h *Holder) Close() error { h.opened.ch = make(chan struct{}) h.opened.mu.Unlock() - h.mu.Lock() - if h.translateStoreReplicator != nil { - h.translateStoreReplicator.Close() - h.translateStoreReplicator = nil - } - h.mu.Unlock() - return nil } @@ -452,9 +438,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { // Update options. h.indexes[index.Name()] = index - // Restart replication. - go h.refreshTranslateStoreReplicator() - return index, nil } @@ -470,7 +453,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data")) index.snapshotQueue = h.snapshotQueue index.holder = h - index.OpenTranslateStore = h.OpenTranslateStore return index, nil } @@ -667,243 +649,6 @@ func (h *Holder) logStartup() error { return nil } -// TranslateStore returns store for the given index or field. -func (h *Holder) TranslateStore(index, field string) (TranslateStore, error) { - if field == "" { - idx := h.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - return idx.TranslateStore(), nil - } - - f := h.Field(index, field) - if f == nil { - return nil, ErrFieldNotFound - } - return f.TranslateStore(), nil -} - -// TranslateOffsetMap returns a map of offsets for all indexes & fields. -func (h *Holder) TranslateOffsetMap() (TranslateOffsetMap, error) { - m := make(TranslateOffsetMap) - for _, idx := range h.Indexes() { - id, err := idx.TranslateStore().MaxID() - if err != nil { - return nil, err - } - m.SetIndexOffset(idx.Name(), id+1) - - for _, field := range idx.Fields() { - id, err := field.TranslateStore().MaxID() - if err != nil { - return nil, err - } - m.SetFieldOffset(idx.Name(), field.Name(), id+1) - } - } - return m, nil -} - -func (h *Holder) setTranslateStoreReadOnly(v bool) { - for _, idx := range h.Indexes() { - idx.TranslateStore().SetReadOnly(v) - for _, field := range idx.Fields() { - field.TranslateStore().SetReadOnly(v) - } - } -} - -func (h *Holder) setPrimaryTranslateStore(node *Node) error { - if node != nil && h.OpenTranslateReader == nil { - return nil - } - h.mu.Lock() - h.primaryTranslateNode = node.Clone() - h.mu.Unlock() - - go h.refreshTranslateStoreReplicator() - return nil -} - -func (h *Holder) refreshTranslateStoreReplicator() { - h.mu.RLock() - node := h.primaryTranslateNode - h.mu.RUnlock() - - var nodeURL string - if node != nil { - u := node.URI.URL() - nodeURL = u.String() - } - - // Stop existing replication, if running. - h.mu.Lock() - if h.translateStoreReplicator != nil { - h.translateStoreReplicator.Close() - h.translateStoreReplicator = nil - } - h.mu.Unlock() - - // Set all stores read only mode based on if we have a primary. - h.setTranslateStoreReadOnly(node != nil) - - // Start replication monitor, if needed. - h.mu.Lock() - defer h.mu.Unlock() - if nodeURL != "" { - h.translateStoreReplicator = newHolderTranslateStoreReplicator(h, nodeURL) - h.translateStoreReplicator.logger = h.Logger - if err := h.translateStoreReplicator.Open(); err != nil { - h.Logger.Printf("cannot open translate store replicator: %s", err) - } - } -} - -// TranslateEntryReader returns a reader that merges all index & field reader -// that are specified in the offsets map. -func (h *Holder) TranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { - // Ensure all readers are cleaned up if any error. - var a []TranslateEntryReader - defer func() { - if err != nil { - for i := range a { - a[i].Close() - } - } - }() - - // Fetch all readers. - for indexName, m := range offsets { - for fieldName, offset := range m { - var store TranslateStore - - idx := h.Index(indexName) - if idx == nil { - return nil, ErrIndexNotFound - } - - // Fetch from index or field store. - if fieldName == "" { - store = idx.TranslateStore() - } else { - f := idx.Field(fieldName) - if f == nil { - return nil, ErrFieldNotFound - } - store = f.TranslateStore() - } - - // Generate reader and append to multireader. - r, err := store.EntryReader(ctx, uint64(offset)) - if err != nil { - return nil, errors.Wrap(err, "translate reader") - } - a = append(a, r) - } - } - - return NewMultiTranslateEntryReader(ctx, a), nil -} - -// holderTranslateStoreReplicator manages the replication of translation store -// data from a primary store to the local replica. Continually tries to -// reconnect on disconnect. -type holderTranslateStoreReplicator struct { - ctx context.Context - cancel func() - wg sync.WaitGroup - - holder *Holder - nodeURL string - - logger logger.Logger -} - -func newHolderTranslateStoreReplicator(h *Holder, nodeURL string) *holderTranslateStoreReplicator { - r := &holderTranslateStoreReplicator{ - holder: h, - nodeURL: nodeURL, - logger: logger.NopLogger, - } - r.ctx, r.cancel = context.WithCancel(context.Background()) - return r -} - -// Open starts the background monitoring goroutine. -func (r *holderTranslateStoreReplicator) Open() error { - r.wg.Add(1) - go func() { defer r.wg.Done(); r.monitor() }() - return nil -} - -// Close stops the replicator. -func (r *holderTranslateStoreReplicator) Close() error { - r.cancel() - return nil -} - -// monitor runs in a background goroutine and continually tries to connect and -// stream translate changes from the primary store. -func (r *holderTranslateStoreReplicator) monitor() { - for { - select { - case <-r.ctx.Done(): - return - default: - if err := r.replicate(); err != nil { - r.logger.Printf("cannot replicate: nodeURL=%s err=%s", r.nodeURL, err) - } - time.Sleep(1 * time.Second) - } - } -} - -func (r *holderTranslateStoreReplicator) replicate() error { - // Determine the offsets of every index & field store. - offsets, err := r.holder.TranslateOffsetMap() - if err != nil { - return err - } else if len(offsets) == 0 { - return nil - } - - // Begin streaming from remote primary. - rd, err := r.holder.OpenTranslateReader(r.ctx, r.nodeURL, offsets) - if err != nil { - return err - } - defer rd.Close() - - for { - var entry TranslateEntry - if err := rd.ReadEntry(&entry); err != nil { - return err - } - - // Find appropriate store. - var store TranslateStore - if entry.Field == "" { - idx := r.holder.Index(entry.Index) - if idx == nil { - return ErrIndexNotFound - } - store = idx.TranslateStore() - } else { - f := r.holder.Field(entry.Index, entry.Field) - if f == nil { - return ErrFieldNotFound - } - store = f.TranslateStore() - } - - // Apply replication to store. - if err := store.ForceSet(entry.ID, entry.Key); err != nil { - return err - } - } -} - // holderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type holderSyncer struct { diff --git a/http/client.go b/http/client.go index 1c5de0197..b99341c15 100644 --- a/http/client.go +++ b/http/client.go @@ -1124,6 +1124,106 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ return errors.Wrap(resp.Body.Close(), "closing response body") } +// TranslateKeysNode sends a key translation request to a specific node. +func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string) ([]uint64, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode") + defer span.Finish() + + if index == "" { + return nil, pilosa.ErrIndexRequired + } + + buf, err := c.serializer.Marshal(&pilosa.TranslateKeysRequest{ + Index: index, + Field: field, + Keys: keys, + }) + if err != nil { + return nil, errors.Wrap(err, "marshaling TranslateKeysRequest") + } + + // Create HTTP request. + u := uri.Path("/internal/translate/keys") + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + tkresp := &pilosa.TranslateKeysResponse{} + if err := c.serializer.Unmarshal(body, tkresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } + return tkresp.IDs, nil +} + +// TranslateIDsNode sends an id translation request to a specific node. +func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode") + defer span.Finish() + + if index == "" { + return nil, pilosa.ErrIndexRequired + } + + buf, err := c.serializer.Marshal(pilosa.TranslateIDsRequest{ + Index: index, + Field: field, + IDs: ids, + }) + if err != nil { + return nil, errors.Wrap(err, "marshaling TranslateIDsRequest") + } + + // Create HTTP request. + u := uri.Path("/internal/translate/ids") + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + tkresp := &pilosa.TranslateIDsResponse{} + if err := c.serializer.Unmarshal(body, tkresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } + return tkresp.Keys, nil +} + // executeRequest executes the given request and checks the Response. For // responses with non-2XX status, the body is read and closed, and an error is // returned. If the error is nil, the caller must ensure that the response body diff --git a/http/handler.go b/http/handler.go index ba7877222..30d296a40 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1752,7 +1752,7 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request return } - buf, err := h.api.TranslateKeys(r.Body) + buf, err := h.api.TranslateKeys(r.Context(), r.Body) if err != nil { http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError) } diff --git a/index.go b/index.go index c96e3bd5f..57533b22a 100644 --- a/index.go +++ b/index.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "sync" "time" @@ -52,8 +53,6 @@ type Index struct { // Column attribute storage and cache. columnAttrs AttrStore - translateStore TranslateStore - broadcaster broadcaster Stats stats.StatsClient @@ -62,9 +61,6 @@ type Index struct { // Used for notifying holder when a field is added. holder *Holder - - // Instantiates new translation stores for fields. - OpenTranslateStore OpenTranslateStoreFunc } // NewIndex returns a new instance of Index. @@ -86,8 +82,6 @@ func NewIndex(path, name string) (*Index, error) { Stats: stats.NopStatsClient, logger: logger.NopLogger, trackExistence: true, - - OpenTranslateStore: OpenInMemTranslateStore, }, nil } @@ -97,15 +91,17 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// TranslateStorePath returns the translation database path for a partition. +func (i *Index) TranslateStorePath(partitionID int) string { + return filepath.Join(i.path, "keys", strconv.Itoa(partitionID)) +} + // Keys returns true if the index uses string keys. func (i *Index) Keys() bool { return i.keys } // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs } -// TranslateStore returns the underlying translation store for the index. -func (i *Index) TranslateStore() TranslateStore { return i.translateStore } - // Options returns all options for this index. func (i *Index) Options() IndexOptions { i.mu.RLock() @@ -149,11 +145,6 @@ func (i *Index) Open() (err error) { return errors.Wrap(err, "opening attrstore") } - // Instantiate & open translation store. - if i.translateStore, err = i.OpenTranslateStore(filepath.Join(i.path, "keys"), i.name, ""); err != nil { - return errors.Wrap(err, "opening translate store") - } - return nil } @@ -279,12 +270,6 @@ func (i *Index) Close() error { } i.fields = make(map[string]*Field) - if i.translateStore != nil { - if err := i.translateStore.Close(); err != nil { - return err - } - } - return nil } @@ -445,11 +430,6 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { // Add to index's field lookup. i.fields[name] = f - // Update replication, if needed. - if i.holder != nil { - go i.holder.refreshTranslateStoreReplicator() - } - return f, nil } @@ -465,7 +445,6 @@ func (i *Index) newField(path, name string) (*Field, error) { if i.snapshotQueue != nil { f.snapshotQueue = i.snapshotQueue } - f.OpenTranslateStore = i.OpenTranslateStore return f, nil } diff --git a/internal/private.pb.go b/internal/private.pb.go index d42e410cf..c941e4dc1 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -3,11 +3,13 @@ package internal -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -import io "io" +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -18,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type IndexMeta struct { Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` @@ -32,7 +34,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{0} + return fileDescriptor_d2a91b51c7bdc125, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -42,15 +44,15 @@ func (m *IndexMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_IndexMeta.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *IndexMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexMeta.Merge(dst, src) +func (m *IndexMeta) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexMeta.Merge(m, src) } func (m *IndexMeta) XXX_Size() int { return m.Size() @@ -96,7 +98,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{1} + return fileDescriptor_d2a91b51c7bdc125, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -106,15 +108,15 @@ func (m *FieldOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_FieldOptions.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *FieldOptions) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldOptions.Merge(dst, src) +func (m *FieldOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOptions.Merge(m, src) } func (m *FieldOptions) XXX_Size() int { return m.Size() @@ -213,7 +215,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{2} + return fileDescriptor_d2a91b51c7bdc125, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -223,15 +225,15 @@ func (m *ImportResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_ImportResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportResponse.Merge(dst, src) +func (m *ImportResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportResponse.Merge(m, src) } func (m *ImportResponse) XXX_Size() int { return m.Size() @@ -264,7 +266,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{3} + return fileDescriptor_d2a91b51c7bdc125, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -274,15 +276,15 @@ func (m *BlockDataRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_BlockDataRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *BlockDataRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataRequest.Merge(dst, src) +func (m *BlockDataRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockDataRequest.Merge(m, src) } func (m *BlockDataRequest) XXX_Size() int { return m.Size() @@ -329,8 +331,8 @@ func (m *BlockDataRequest) GetBlock() uint64 { } type BlockDataResponse struct { - RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -340,7 +342,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{4} + return fileDescriptor_d2a91b51c7bdc125, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -350,15 +352,15 @@ func (m *BlockDataResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_BlockDataResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *BlockDataResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockDataResponse.Merge(dst, src) +func (m *BlockDataResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockDataResponse.Merge(m, src) } func (m *BlockDataResponse) XXX_Size() int { return m.Size() @@ -384,7 +386,7 @@ func (m *BlockDataResponse) GetColumnIDs() []uint64 { } type Cache struct { - IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs,proto3" json:"IDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -394,7 +396,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{5} + return fileDescriptor_d2a91b51c7bdc125, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -404,15 +406,15 @@ func (m *Cache) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Cache.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Cache) XXX_Merge(src proto.Message) { - xxx_messageInfo_Cache.Merge(dst, src) +func (m *Cache) XXX_Merge(src proto.Message) { + xxx_messageInfo_Cache.Merge(m, src) } func (m *Cache) XXX_Size() int { return m.Size() @@ -431,7 +433,7 @@ func (m *Cache) GetIDs() []uint64 { } type MaxShards struct { - Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard,proto3" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -441,7 +443,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{6} + return fileDescriptor_d2a91b51c7bdc125, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -451,15 +453,15 @@ func (m *MaxShards) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MaxShards.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *MaxShards) XXX_Merge(src proto.Message) { - xxx_messageInfo_MaxShards.Merge(dst, src) +func (m *MaxShards) XXX_Merge(src proto.Message) { + xxx_messageInfo_MaxShards.Merge(m, src) } func (m *MaxShards) XXX_Size() int { return m.Size() @@ -490,7 +492,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{7} + return fileDescriptor_d2a91b51c7bdc125, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -500,15 +502,15 @@ func (m *CreateShardMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateShardMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *CreateShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateShardMessage.Merge(dst, src) +func (m *CreateShardMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateShardMessage.Merge(m, src) } func (m *CreateShardMessage) XXX_Size() int { return m.Size() @@ -551,7 +553,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{8} + return fileDescriptor_d2a91b51c7bdc125, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -561,15 +563,15 @@ func (m *DeleteIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_DeleteIndexMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *DeleteIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteIndexMessage.Merge(dst, src) +func (m *DeleteIndexMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteIndexMessage.Merge(m, src) } func (m *DeleteIndexMessage) XXX_Size() int { return m.Size() @@ -589,7 +591,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -599,7 +601,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{9} + return fileDescriptor_d2a91b51c7bdc125, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -609,15 +611,15 @@ func (m *CreateIndexMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateIndexMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *CreateIndexMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateIndexMessage.Merge(dst, src) +func (m *CreateIndexMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateIndexMessage.Merge(m, src) } func (m *CreateIndexMessage) XXX_Size() int { return m.Size() @@ -645,7 +647,7 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -655,7 +657,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{10} + return fileDescriptor_d2a91b51c7bdc125, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -665,15 +667,15 @@ func (m *CreateFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_CreateFieldMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *CreateFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateFieldMessage.Merge(dst, src) +func (m *CreateFieldMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateFieldMessage.Merge(m, src) } func (m *CreateFieldMessage) XXX_Size() int { return m.Size() @@ -717,7 +719,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{11} + return fileDescriptor_d2a91b51c7bdc125, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -727,15 +729,15 @@ func (m *DeleteFieldMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_DeleteFieldMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *DeleteFieldMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteFieldMessage.Merge(dst, src) +func (m *DeleteFieldMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteFieldMessage.Merge(m, src) } func (m *DeleteFieldMessage) XXX_Size() int { return m.Size() @@ -773,7 +775,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{12} + return fileDescriptor_d2a91b51c7bdc125, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -783,15 +785,15 @@ func (m *DeleteAvailableShardMessage) XXX_Marshal(b []byte, deterministic bool) return xxx_messageInfo_DeleteAvailableShardMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteAvailableShardMessage.Merge(dst, src) +func (m *DeleteAvailableShardMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteAvailableShardMessage.Merge(m, src) } func (m *DeleteAvailableShardMessage) XXX_Size() int { return m.Size() @@ -825,8 +827,8 @@ func (m *DeleteAvailableShardMessage) GetShardID() uint64 { type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -836,7 +838,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{13} + return fileDescriptor_d2a91b51c7bdc125, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -846,15 +848,15 @@ func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) +func (m *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(m, src) } func (m *Field) XXX_Size() int { return m.Size() @@ -887,7 +889,7 @@ func (m *Field) GetViews() []string { } type Schema struct { - Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes" json:"Indexes,omitempty"` + Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -897,7 +899,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{14} + return fileDescriptor_d2a91b51c7bdc125, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,15 +909,15 @@ func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) +func (m *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(m, src) } func (m *Schema) XXX_Size() int { return m.Size() @@ -935,7 +937,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -945,7 +947,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{15} + return fileDescriptor_d2a91b51c7bdc125, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -955,15 +957,15 @@ func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Index.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Index) XXX_Merge(src proto.Message) { - xxx_messageInfo_Index.Merge(dst, src) +func (m *Index) XXX_Merge(src proto.Message) { + xxx_messageInfo_Index.Merge(m, src) } func (m *Index) XXX_Size() int { return m.Size() @@ -1001,7 +1003,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{16} + return fileDescriptor_d2a91b51c7bdc125, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1011,15 +1013,15 @@ func (m *URI) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_URI.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *URI) XXX_Merge(src proto.Message) { - xxx_messageInfo_URI.Merge(dst, src) +func (m *URI) XXX_Merge(src proto.Message) { + xxx_messageInfo_URI.Merge(m, src) } func (m *URI) XXX_Size() int { return m.Size() @@ -1053,7 +1055,7 @@ func (m *URI) GetPort() uint32 { type Node struct { ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - URI *URI `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"` + URI *URI `protobuf:"bytes,2,opt,name=URI,proto3" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1065,7 +1067,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{17} + return fileDescriptor_d2a91b51c7bdc125, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1075,15 +1077,15 @@ func (m *Node) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Node.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Node) XXX_Merge(src proto.Message) { - xxx_messageInfo_Node.Merge(dst, src) +func (m *Node) XXX_Merge(src proto.Message) { + xxx_messageInfo_Node.Merge(m, src) } func (m *Node) XXX_Size() int { return m.Size() @@ -1134,7 +1136,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{18} + return fileDescriptor_d2a91b51c7bdc125, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1144,15 +1146,15 @@ func (m *NodeStateMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_NodeStateMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *NodeStateMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStateMessage.Merge(dst, src) +func (m *NodeStateMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStateMessage.Merge(m, src) } func (m *NodeStateMessage) XXX_Size() int { return m.Size() @@ -1179,7 +1181,7 @@ func (m *NodeStateMessage) GetState() string { type NodeEventMessage struct { Event uint32 `protobuf:"varint,1,opt,name=Event,proto3" json:"Event,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1189,7 +1191,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{19} + return fileDescriptor_d2a91b51c7bdc125, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1199,15 +1201,15 @@ func (m *NodeEventMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_NodeEventMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *NodeEventMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeEventMessage.Merge(dst, src) +func (m *NodeEventMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeEventMessage.Merge(m, src) } func (m *NodeEventMessage) XXX_Size() int { return m.Size() @@ -1233,9 +1235,9 @@ func (m *NodeEventMessage) GetNode() *Node { } type NodeStatus struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` - Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes" json:"Indexes,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node,proto3" json:"Node,omitempty"` + Schema *Schema `protobuf:"bytes,3,opt,name=Schema,proto3" json:"Schema,omitempty"` + Indexes []*IndexStatus `protobuf:"bytes,4,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1245,7 +1247,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{20} + return fileDescriptor_d2a91b51c7bdc125, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1255,15 +1257,15 @@ func (m *NodeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *NodeStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStatus.Merge(dst, src) +func (m *NodeStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStatus.Merge(m, src) } func (m *NodeStatus) XXX_Size() int { return m.Size() @@ -1297,7 +1299,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields" json:"Fields,omitempty"` + Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1307,7 +1309,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{21} + return fileDescriptor_d2a91b51c7bdc125, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1317,15 +1319,15 @@ func (m *IndexStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_IndexStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *IndexStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexStatus.Merge(dst, src) +func (m *IndexStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexStatus.Merge(m, src) } func (m *IndexStatus) XXX_Size() int { return m.Size() @@ -1352,7 +1354,7 @@ func (m *IndexStatus) GetFields() []*FieldStatus { type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards" json:"AvailableShards,omitempty"` + AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1362,7 +1364,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{22} + return fileDescriptor_d2a91b51c7bdc125, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1372,15 +1374,15 @@ func (m *FieldStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_FieldStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *FieldStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldStatus.Merge(dst, src) +func (m *FieldStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldStatus.Merge(m, src) } func (m *FieldStatus) XXX_Size() int { return m.Size() @@ -1408,7 +1410,7 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes" json:"Nodes,omitempty"` + Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1418,7 +1420,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{23} + return fileDescriptor_d2a91b51c7bdc125, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1428,15 +1430,15 @@ func (m *ClusterStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ClusterStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ClusterStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClusterStatus.Merge(dst, src) +func (m *ClusterStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClusterStatus.Merge(m, src) } func (m *ClusterStatus) XXX_Size() int { return m.Size() @@ -1482,7 +1484,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{24} + return fileDescriptor_d2a91b51c7bdc125, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1492,15 +1494,15 @@ func (m *BSIGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BSIGroup.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *BSIGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_BSIGroup.Merge(dst, src) +func (m *BSIGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_BSIGroup.Merge(m, src) } func (m *BSIGroup) XXX_Size() int { return m.Size() @@ -1552,7 +1554,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{25} + return fileDescriptor_d2a91b51c7bdc125, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1562,15 +1564,15 @@ func (m *CreateViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_CreateViewMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *CreateViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateViewMessage.Merge(dst, src) +func (m *CreateViewMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateViewMessage.Merge(m, src) } func (m *CreateViewMessage) XXX_Size() int { return m.Size() @@ -1615,7 +1617,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{26} + return fileDescriptor_d2a91b51c7bdc125, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1625,15 +1627,15 @@ func (m *DeleteViewMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_DeleteViewMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *DeleteViewMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteViewMessage.Merge(dst, src) +func (m *DeleteViewMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteViewMessage.Merge(m, src) } func (m *DeleteViewMessage) XXX_Size() int { return m.Size() @@ -1667,11 +1669,11 @@ func (m *DeleteViewMessage) GetView() string { type ResizeInstruction struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` - Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` - Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` - ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` + Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator,proto3" json:"Coordinator,omitempty"` + Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources,proto3" json:"Sources,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus,proto3" json:"NodeStatus,omitempty"` + ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus,proto3" json:"ClusterStatus,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1681,7 +1683,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{27} + return fileDescriptor_d2a91b51c7bdc125, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1691,15 +1693,15 @@ func (m *ResizeInstruction) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_ResizeInstruction.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ResizeInstruction) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstruction.Merge(dst, src) +func (m *ResizeInstruction) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeInstruction.Merge(m, src) } func (m *ResizeInstruction) XXX_Size() int { return m.Size() @@ -1753,7 +1755,7 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { } type ResizeSource struct { - Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,1,opt,name=Node,proto3" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` @@ -1767,7 +1769,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{28} + return fileDescriptor_d2a91b51c7bdc125, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1777,15 +1779,15 @@ func (m *ResizeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_ResizeSource.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ResizeSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeSource.Merge(dst, src) +func (m *ResizeSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeSource.Merge(m, src) } func (m *ResizeSource) XXX_Size() int { return m.Size() @@ -1833,7 +1835,7 @@ func (m *ResizeSource) GetShard() uint64 { type ResizeInstructionComplete struct { JobID int64 `protobuf:"varint,1,opt,name=JobID,proto3" json:"JobID,omitempty"` - Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` + Node *Node `protobuf:"bytes,2,opt,name=Node,proto3" json:"Node,omitempty"` Error string `protobuf:"bytes,3,opt,name=Error,proto3" json:"Error,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1844,7 +1846,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{29} + return fileDescriptor_d2a91b51c7bdc125, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1854,15 +1856,15 @@ func (m *ResizeInstructionComplete) XXX_Marshal(b []byte, deterministic bool) ([ return xxx_messageInfo_ResizeInstructionComplete.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ResizeInstructionComplete) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResizeInstructionComplete.Merge(dst, src) +func (m *ResizeInstructionComplete) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeInstructionComplete.Merge(m, src) } func (m *ResizeInstructionComplete) XXX_Size() int { return m.Size() @@ -1895,7 +1897,7 @@ func (m *ResizeInstructionComplete) GetError() string { } type SetCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` + New *Node `protobuf:"bytes,1,opt,name=New,proto3" json:"New,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1905,7 +1907,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{30} + return fileDescriptor_d2a91b51c7bdc125, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1915,15 +1917,15 @@ func (m *SetCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([]byt return xxx_messageInfo_SetCoordinatorMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *SetCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_SetCoordinatorMessage.Merge(dst, src) +func (m *SetCoordinatorMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetCoordinatorMessage.Merge(m, src) } func (m *SetCoordinatorMessage) XXX_Size() int { return m.Size() @@ -1942,7 +1944,7 @@ func (m *SetCoordinatorMessage) GetNew() *Node { } type UpdateCoordinatorMessage struct { - New *Node `protobuf:"bytes,1,opt,name=New" json:"New,omitempty"` + New *Node `protobuf:"bytes,1,opt,name=New,proto3" json:"New,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1952,7 +1954,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{31} + return fileDescriptor_d2a91b51c7bdc125, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1962,15 +1964,15 @@ func (m *UpdateCoordinatorMessage) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_UpdateCoordinatorMessage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_UpdateCoordinatorMessage.Merge(dst, src) +func (m *UpdateCoordinatorMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateCoordinatorMessage.Merge(m, src) } func (m *UpdateCoordinatorMessage) XXX_Size() int { return m.Size() @@ -1990,7 +1992,7 @@ func (m *UpdateCoordinatorMessage) GetNew() *Node { type Topology struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` - NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs" json:"NodeIDs,omitempty"` + NodeIDs []string `protobuf:"bytes,2,rep,name=NodeIDs,proto3" json:"NodeIDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2000,7 +2002,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{32} + return fileDescriptor_d2a91b51c7bdc125, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2010,15 +2012,15 @@ func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Topology.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Topology) XXX_Merge(src proto.Message) { - xxx_messageInfo_Topology.Merge(dst, src) +func (m *Topology) XXX_Merge(src proto.Message) { + xxx_messageInfo_Topology.Merge(m, src) } func (m *Topology) XXX_Size() int { return m.Size() @@ -2053,7 +2055,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_e6d12fddb5948a73, []int{33} + return fileDescriptor_d2a91b51c7bdc125, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2063,15 +2065,15 @@ func (m *RecalculateCaches) XXX_Marshal(b []byte, deterministic bool) ([]byte, e return xxx_messageInfo_RecalculateCaches.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *RecalculateCaches) XXX_Merge(src proto.Message) { - xxx_messageInfo_RecalculateCaches.Merge(dst, src) +func (m *RecalculateCaches) XXX_Merge(src proto.Message) { + xxx_messageInfo_RecalculateCaches.Merge(m, src) } func (m *RecalculateCaches) XXX_Size() int { return m.Size() @@ -2119,10 +2121,91 @@ func init() { proto.RegisterType((*Topology)(nil), "internal.Topology") proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") } + +func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } + +var fileDescriptor_d2a91b51c7bdc125 = []byte{ + // 1174 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x72, 0xdb, 0x44, + 0x18, 0x46, 0x87, 0x38, 0xf6, 0xef, 0x38, 0x07, 0xb5, 0x0d, 0x6a, 0x61, 0x82, 0xd9, 0xe9, 0x50, + 0xd3, 0x19, 0x42, 0xa7, 0xe5, 0x82, 0x53, 0x67, 0x8a, 0xe3, 0x50, 0x44, 0x49, 0x28, 0xeb, 0x24, + 0x77, 0x5c, 0x6c, 0xec, 0x9d, 0x46, 0x13, 0x59, 0x32, 0xd2, 0x2a, 0x89, 0x7b, 0xc1, 0x2d, 0xcc, + 0xf0, 0x02, 0x3c, 0x41, 0x9f, 0x85, 0x4b, 0x1e, 0x81, 0x09, 0x2f, 0xc2, 0xec, 0xbf, 0xbb, 0x92, + 0xec, 0xb8, 0x24, 0x84, 0xde, 0xed, 0xff, 0xfd, 0xfb, 0x9f, 0x0f, 0x5a, 0x41, 0x6b, 0x9c, 0x86, + 0x27, 0x4c, 0xf0, 0xcd, 0x71, 0x9a, 0x88, 0xc4, 0xab, 0x87, 0xb1, 0xe0, 0x69, 0xcc, 0x22, 0xf2, + 0x14, 0x1a, 0x41, 0x3c, 0xe4, 0x67, 0x3b, 0x5c, 0x30, 0xcf, 0x03, 0xf7, 0x19, 0x9f, 0x64, 0xbe, + 0xd3, 0xb6, 0x3a, 0x75, 0x8a, 0x67, 0xef, 0x03, 0x58, 0xde, 0x4b, 0xd9, 0xe0, 0x78, 0xfb, 0x2c, + 0xcc, 0x04, 0x8f, 0x07, 0xdc, 0x77, 0x91, 0x3b, 0x83, 0x92, 0x57, 0x36, 0x2c, 0x7d, 0x1d, 0xf2, + 0x68, 0xf8, 0xfd, 0x58, 0x84, 0x49, 0x9c, 0x49, 0x65, 0x7b, 0x93, 0x31, 0xf7, 0xeb, 0x6d, 0xab, + 0xd3, 0xa0, 0x78, 0xf6, 0xde, 0x85, 0xc6, 0x16, 0x1b, 0x1c, 0x71, 0x64, 0x38, 0xc8, 0x28, 0x81, + 0x82, 0xdb, 0x0f, 0x5f, 0x2a, 0x2b, 0x2d, 0x5a, 0x02, 0x5e, 0x1b, 0x9a, 0x7b, 0xe1, 0x88, 0xff, + 0x90, 0xb3, 0x58, 0xe4, 0x23, 0x7f, 0x01, 0xa5, 0xab, 0x90, 0xb7, 0x0a, 0xce, 0x4e, 0x18, 0xfb, + 0x8d, 0xb6, 0xd5, 0x71, 0xa8, 0x3c, 0x22, 0xc2, 0xce, 0x7c, 0xd0, 0x08, 0x3b, 0x2b, 0x42, 0x6c, + 0x4e, 0x87, 0xb8, 0x9b, 0xf4, 0x05, 0x8b, 0x87, 0x2c, 0x1d, 0x1e, 0x84, 0xfc, 0xd4, 0x5f, 0x52, + 0x21, 0x4e, 0xa3, 0x52, 0xb6, 0xcb, 0x32, 0xee, 0xb7, 0x50, 0x1d, 0x9e, 0xbd, 0x3b, 0x50, 0xef, + 0x86, 0xa2, 0xc7, 0xc7, 0xe2, 0xc8, 0x5f, 0x6e, 0x5b, 0x1d, 0x97, 0x16, 0xb4, 0x77, 0x13, 0x16, + 0xfa, 0x03, 0x16, 0x71, 0x7f, 0x05, 0x05, 0x14, 0x41, 0x08, 0x2c, 0x07, 0xa3, 0x71, 0x92, 0x0a, + 0xca, 0xb3, 0x71, 0x12, 0x67, 0x5c, 0x7a, 0xb9, 0x9d, 0xa6, 0xbe, 0x85, 0x11, 0xc9, 0x23, 0xf9, + 0x19, 0x56, 0xbb, 0x51, 0x32, 0x38, 0xee, 0x31, 0xc1, 0x28, 0xff, 0x29, 0xe7, 0x99, 0x90, 0xda, + 0xb0, 0x52, 0xfa, 0x9e, 0x22, 0x24, 0x8a, 0x59, 0xf7, 0x6d, 0x85, 0x22, 0x21, 0x3d, 0xc5, 0x38, + 0x54, 0x92, 0xf0, 0x8c, 0xde, 0x1c, 0xb1, 0x74, 0x88, 0x99, 0x75, 0xa9, 0x22, 0x24, 0x8a, 0x96, + 0xb0, 0x1a, 0x2e, 0x55, 0x04, 0x09, 0x60, 0xad, 0x62, 0x5f, 0xbb, 0xb9, 0x0e, 0x35, 0x9a, 0x9c, + 0x06, 0xbd, 0xcc, 0xb7, 0xda, 0x4e, 0xc7, 0xa5, 0x9a, 0xc2, 0xb2, 0x25, 0x51, 0x3e, 0x8a, 0x25, + 0xcb, 0x46, 0x56, 0x09, 0x90, 0xdb, 0xb0, 0x80, 0x35, 0x94, 0x51, 0x96, 0xb2, 0xf2, 0x48, 0x7e, + 0xb1, 0xa0, 0xb1, 0xc3, 0xce, 0xd0, 0x91, 0xcc, 0x7b, 0x0c, 0x75, 0x93, 0x6d, 0xbc, 0xd4, 0x7c, + 0xf8, 0xfe, 0xa6, 0x69, 0xd3, 0xcd, 0xe2, 0xda, 0xa6, 0xb9, 0xb3, 0x1d, 0x8b, 0x74, 0x42, 0x0b, + 0x91, 0x3b, 0x5f, 0x40, 0x6b, 0x8a, 0x25, 0xed, 0x1d, 0xf3, 0x89, 0xc9, 0xea, 0x31, 0x9f, 0xc8, + 0x58, 0x4f, 0x58, 0x94, 0x73, 0xcc, 0x95, 0x4b, 0x15, 0xf1, 0xb9, 0xfd, 0xa9, 0x45, 0x0e, 0xc0, + 0xdb, 0x4a, 0x39, 0x13, 0x1c, 0x8d, 0xec, 0xf0, 0x2c, 0x63, 0x2f, 0xf8, 0x65, 0x19, 0x77, 0xaa, + 0x19, 0x2f, 0xb2, 0x6b, 0x57, 0xb2, 0x4b, 0xee, 0x83, 0xd7, 0xe3, 0x11, 0x17, 0x5c, 0xcf, 0xd8, + 0xbf, 0xe8, 0x25, 0x7d, 0xe3, 0xc3, 0xe5, 0x77, 0xbd, 0x7b, 0xe0, 0xca, 0x81, 0x45, 0x63, 0xcd, + 0x87, 0x37, 0xca, 0x3c, 0x15, 0xb3, 0x4c, 0xf1, 0x02, 0x89, 0x8c, 0x52, 0xf4, 0xf2, 0x8a, 0x81, + 0x4d, 0xb5, 0xd2, 0x7d, 0x6d, 0xca, 0x41, 0x53, 0xeb, 0xa5, 0xa9, 0xea, 0xb0, 0x6b, 0x6b, 0x4f, + 0x4c, 0xb8, 0xd7, 0xb5, 0x46, 0x06, 0xf0, 0x8e, 0xd2, 0xf0, 0xd5, 0x09, 0x0b, 0x23, 0x76, 0x18, + 0xfd, 0xa7, 0x8a, 0x4c, 0x39, 0xee, 0xc3, 0x22, 0xca, 0x06, 0x3d, 0xdd, 0xdb, 0x86, 0x24, 0x3f, + 0x42, 0x39, 0x26, 0xbb, 0x6c, 0xc4, 0xb5, 0x36, 0x3c, 0x17, 0xf1, 0xda, 0x97, 0xc7, 0x2b, 0x0d, + 0xcb, 0xd1, 0x92, 0x0b, 0xd3, 0x91, 0x86, 0x91, 0x20, 0x8f, 0xa0, 0xd6, 0x1f, 0x1c, 0xf1, 0x11, + 0xf3, 0x3e, 0x84, 0x45, 0xf4, 0x90, 0x67, 0xba, 0xa3, 0x57, 0x66, 0x2a, 0x45, 0x0d, 0x9f, 0xf4, + 0x74, 0x64, 0x73, 0x7d, 0xba, 0x07, 0x35, 0xb4, 0x9e, 0xf9, 0xee, 0xac, 0x1a, 0xc4, 0xa9, 0x66, + 0x93, 0x6d, 0x70, 0xf6, 0x69, 0x20, 0x27, 0x15, 0x3d, 0x30, 0x5a, 0x34, 0x25, 0x75, 0x7f, 0x93, + 0x64, 0x42, 0xe7, 0x09, 0xcf, 0x12, 0x7b, 0x9e, 0xa4, 0x02, 0x73, 0xd4, 0xa2, 0x78, 0x26, 0x19, + 0xb8, 0xbb, 0xc9, 0x90, 0x7b, 0xcb, 0x60, 0x07, 0x3d, 0xad, 0xc3, 0x0e, 0x7a, 0xde, 0x7b, 0xa8, + 0x5e, 0xa7, 0xa6, 0x55, 0x3a, 0xb1, 0x4f, 0x03, 0x8a, 0x86, 0xef, 0x42, 0x2b, 0xc8, 0xb6, 0x92, + 0x24, 0x1d, 0x86, 0x31, 0x13, 0x49, 0xaa, 0xbf, 0x24, 0xd3, 0x20, 0xce, 0x8a, 0x60, 0x42, 0xed, + 0xf8, 0x06, 0x55, 0x04, 0x79, 0x02, 0xab, 0xd2, 0x28, 0x12, 0xa6, 0xde, 0xeb, 0x50, 0x93, 0x58, + 0xe1, 0x84, 0xa6, 0x4a, 0x0d, 0x76, 0x55, 0xc3, 0x77, 0x4a, 0xc3, 0xf6, 0x09, 0x8f, 0x45, 0xa5, + 0x63, 0x90, 0x46, 0x05, 0x2d, 0xaa, 0x08, 0x8f, 0xa8, 0x00, 0x75, 0x24, 0xcb, 0x65, 0x24, 0x12, + 0xa5, 0xc8, 0x23, 0xbf, 0x59, 0x00, 0xc6, 0xa1, 0x3c, 0x2b, 0x44, 0xac, 0xd7, 0x8b, 0x78, 0x1d, + 0x53, 0x79, 0x3d, 0x2d, 0xab, 0xe5, 0x2d, 0x85, 0x53, 0xd3, 0x19, 0x1f, 0x97, 0x9d, 0xa1, 0x4a, + 0x7a, 0x6b, 0xa6, 0x33, 0x94, 0xd5, 0xb2, 0x3f, 0x9e, 0x43, 0xb3, 0x82, 0xcf, 0xed, 0x92, 0x8f, + 0x8a, 0x2e, 0xb1, 0x67, 0x55, 0x22, 0xae, 0x55, 0x9a, 0x5e, 0x79, 0x06, 0xcd, 0x0a, 0x3c, 0x57, + 0x63, 0x07, 0x56, 0xa6, 0xe7, 0xd0, 0xec, 0xf7, 0x59, 0x98, 0x84, 0xd0, 0xda, 0x8a, 0xf2, 0x4c, + 0xf0, 0x54, 0xab, 0x93, 0x1f, 0x05, 0x05, 0x14, 0xc5, 0x2b, 0x81, 0xf9, 0xf5, 0xf3, 0xee, 0xc2, + 0x82, 0x4c, 0xa3, 0x1a, 0xa7, 0x8b, 0x39, 0x56, 0x4c, 0x72, 0x00, 0xf5, 0x6e, 0x3f, 0x78, 0x9a, + 0x26, 0xf9, 0x78, 0xae, 0xd3, 0xe6, 0xdd, 0x61, 0x57, 0xde, 0x1d, 0xfa, 0x65, 0xe0, 0x5c, 0x78, + 0x19, 0xb8, 0xc5, 0xcb, 0x80, 0xf4, 0x61, 0x4d, 0xad, 0x4a, 0x39, 0xc5, 0xd7, 0x59, 0x38, 0xe6, + 0xa3, 0xeb, 0x94, 0x1f, 0x5d, 0xa9, 0x54, 0xed, 0xb3, 0x37, 0xa9, 0xf4, 0x95, 0x0d, 0x6b, 0x94, + 0x67, 0xe1, 0x4b, 0x1e, 0xc4, 0x99, 0x48, 0xf3, 0x81, 0xdc, 0x49, 0x52, 0xfe, 0xdb, 0xe4, 0x50, + 0x67, 0xdb, 0xa1, 0x8a, 0xb8, 0x4a, 0xa7, 0x7b, 0x0f, 0xa0, 0x39, 0x3b, 0xb3, 0x17, 0xaf, 0x56, + 0xaf, 0x78, 0x0f, 0x60, 0xb1, 0x9f, 0xe4, 0xe9, 0xa0, 0x68, 0xdf, 0xca, 0x9e, 0x54, 0x9e, 0x29, + 0x36, 0x35, 0xd7, 0xbc, 0x4f, 0xaa, 0xc3, 0xe4, 0x2f, 0xa2, 0x89, 0x9b, 0xd3, 0x26, 0x74, 0x7f, + 0x56, 0x87, 0xee, 0xf1, 0x4c, 0x5b, 0xf9, 0x35, 0x14, 0x7c, 0xbb, 0x14, 0x9c, 0x62, 0xd3, 0xe9, + 0xdb, 0xe4, 0x57, 0x0b, 0x96, 0xaa, 0xee, 0x5c, 0x69, 0x88, 0x8b, 0xea, 0xd8, 0x97, 0x7f, 0xf5, + 0x4d, 0x75, 0xdc, 0x79, 0xef, 0xac, 0x85, 0xea, 0x4b, 0xe0, 0x18, 0x6e, 0x5f, 0x28, 0xd9, 0x56, + 0x32, 0x1a, 0xcb, 0xde, 0xf8, 0x1f, 0xa5, 0x93, 0xeb, 0x2d, 0x4d, 0x75, 0xd1, 0x1a, 0x54, 0x11, + 0xe4, 0x33, 0xb8, 0xd5, 0xe7, 0xa2, 0x52, 0x30, 0xd3, 0x79, 0x6d, 0x70, 0x76, 0xf9, 0xe9, 0x6b, + 0xc2, 0x97, 0x2c, 0xf2, 0x25, 0xf8, 0xfb, 0xe3, 0x21, 0x13, 0xfc, 0x5a, 0xd2, 0x5d, 0xa8, 0xef, + 0x25, 0xe3, 0x24, 0x4a, 0x5e, 0x4c, 0x2e, 0xd9, 0x00, 0x3e, 0x2c, 0xaa, 0x5d, 0xae, 0x56, 0x4a, + 0x83, 0x1a, 0x92, 0xdc, 0x90, 0xcd, 0x3d, 0x60, 0xd1, 0x20, 0x8f, 0xa4, 0x1b, 0xf2, 0xed, 0x98, + 0x75, 0x57, 0xff, 0x38, 0xdf, 0xb0, 0xfe, 0x3c, 0xdf, 0xb0, 0xfe, 0x3a, 0xdf, 0xb0, 0x7e, 0xff, + 0x7b, 0xe3, 0xad, 0xc3, 0x1a, 0xfe, 0xc9, 0x3c, 0xfa, 0x27, 0x00, 0x00, 0xff, 0xff, 0xe1, 0xbf, + 0xc3, 0x49, 0xda, 0x0c, 0x00, 0x00, +} + func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2130,40 +2213,46 @@ func (m *IndexMeta) Marshal() (dAtA []byte, err error) { } func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IndexMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Keys { - dAtA[i] = 0x18 - i++ - if m.Keys { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.TrackExistence { - dAtA[i] = 0x20 - i++ + i-- if m.TrackExistence { dAtA[i] = 1 } else { dAtA[i] = 0 } - i++ + i-- + dAtA[i] = 0x20 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Keys { + i-- + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 } - return i, nil + return len(dAtA) - i, nil } func (m *FieldOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2171,88 +2260,97 @@ func (m *FieldOptions) Marshal() (dAtA []byte, err error) { } func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.CacheType) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) - i += copy(dAtA[i:], m.CacheType) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.CacheSize != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) + if m.Scale != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Scale)) + i-- + dAtA[i] = 0x78 } - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0x2a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) + if m.BitDepth != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) + i-- + dAtA[i] = 0x70 } - if len(m.Type) > 0 { - dAtA[i] = 0x42 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) - i += copy(dAtA[i:], m.Type) - } - if m.Min != 0 { - dAtA[i] = 0x48 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) - } - if m.Max != 0 { - dAtA[i] = 0x50 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) - } - if m.Keys { - dAtA[i] = 0x58 - i++ - if m.Keys { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ + if m.Base != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) + i-- + dAtA[i] = 0x68 } if m.NoStandardView { - dAtA[i] = 0x60 - i++ + i-- if m.NoStandardView { dAtA[i] = 1 } else { dAtA[i] = 0 } - i++ + i-- + dAtA[i] = 0x60 } - if m.Base != 0 { - dAtA[i] = 0x68 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Base)) + if m.Keys { + i-- + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x58 } - if m.BitDepth != 0 { - dAtA[i] = 0x70 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.BitDepth)) + if m.Max != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x50 } - if m.Scale != 0 { - dAtA[i] = 0x78 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Scale)) + if m.Min != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + i-- + dAtA[i] = 0x48 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Type) > 0 { + i -= len(m.Type) + copy(dAtA[i:], m.Type) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i-- + dAtA[i] = 0x42 } - return i, nil + if len(m.TimeQuantum) > 0 { + i -= len(m.TimeQuantum) + copy(dAtA[i:], m.TimeQuantum) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i-- + dAtA[i] = 0x2a + } + if m.CacheSize != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) + i-- + dAtA[i] = 0x20 + } + if len(m.CacheType) > 0 { + i -= len(m.CacheType) + copy(dAtA[i:], m.CacheType) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil } func (m *ImportResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2260,26 +2358,33 @@ func (m *ImportResponse) Marshal() (dAtA []byte, err error) { } func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Err) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) - i += copy(dAtA[i:], m.Err) - } if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + if len(m.Err) > 0 { + i -= len(m.Err) + copy(dAtA[i:], m.Err) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Err))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2287,48 +2392,57 @@ func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { } func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlockDataRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } - if m.Block != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) - } - if m.Shard != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.View) > 0 { - dAtA[i] = 0x2a - i++ + i -= len(m.View) + copy(dAtA[i:], m.View) i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i += copy(dAtA[i:], m.View) + i-- + dAtA[i] = 0x2a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Shard != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x20 } - return i, nil + if m.Block != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) + i-- + dAtA[i] = 0x18 + } + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2336,14 +2450,23 @@ func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { } func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlockDataResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.RowIDs) > 0 { - dAtA2 := make([]byte, len(m.RowIDs)*10) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.ColumnIDs) > 0 { + dAtA2 := make([]byte, len(m.ColumnIDs)*10) var j1 int - for _, num := range m.RowIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2352,15 +2475,16 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { dAtA2[j1] = uint8(num) j1++ } - dAtA[i] = 0xa - i++ + i -= j1 + copy(dAtA[i:], dAtA2[:j1]) i = encodeVarintPrivate(dAtA, i, uint64(j1)) - i += copy(dAtA[i:], dAtA2[:j1]) + i-- + dAtA[i] = 0x12 } - if len(m.ColumnIDs) > 0 { - dAtA4 := make([]byte, len(m.ColumnIDs)*10) + if len(m.RowIDs) > 0 { + dAtA4 := make([]byte, len(m.RowIDs)*10) var j3 int - for _, num := range m.ColumnIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2369,21 +2493,19 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { dAtA4[j3] = uint8(num) j3++ } - dAtA[i] = 0x12 - i++ + i -= j3 + copy(dAtA[i:], dAtA4[:j3]) i = encodeVarintPrivate(dAtA, i, uint64(j3)) - i += copy(dAtA[i:], dAtA4[:j3]) + i-- + dAtA[i] = 0xa } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *Cache) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2391,10 +2513,19 @@ func (m *Cache) Marshal() (dAtA []byte, err error) { } func (m *Cache) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Cache) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.IDs) > 0 { dAtA6 := make([]byte, len(m.IDs)*10) var j5 int @@ -2407,21 +2538,19 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { dAtA6[j5] = uint8(num) j5++ } - dAtA[i] = 0xa - i++ + i -= j5 + copy(dAtA[i:], dAtA6[:j5]) i = encodeVarintPrivate(dAtA, i, uint64(j5)) - i += copy(dAtA[i:], dAtA6[:j5]) + i-- + dAtA[i] = 0xa } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *MaxShards) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2429,36 +2558,43 @@ func (m *MaxShards) Marshal() (dAtA []byte, err error) { } func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MaxShards) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.Standard) > 0 { - for k, _ := range m.Standard { - dAtA[i] = 0xa - i++ + for k := range m.Standard { v := m.Standard[k] - mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) - i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(k))) - i += copy(dAtA[i:], k) - dAtA[i] = 0x10 - i++ + baseI := i i = encodeVarintPrivate(dAtA, i, uint64(v)) + i-- + dAtA[i] = 0x10 + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2466,37 +2602,45 @@ func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateShardMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if m.Shard != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Field) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Field) + copy(dAtA[i:], m.Field) i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Shard != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x10 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2504,26 +2648,33 @@ func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DeleteIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2531,36 +2682,45 @@ func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Meta != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n7, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n7 + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2568,42 +2728,52 @@ func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Meta != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n8, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n8 + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2611,32 +2781,40 @@ func (m *DeleteFieldMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DeleteFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ + i -= len(m.Field) + copy(dAtA[i:], m.Field) i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *DeleteAvailableShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2644,37 +2822,45 @@ func (m *DeleteAvailableShardMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteAvailableShardMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DeleteAvailableShardMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.ShardID != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2682,51 +2868,54 @@ func (m *Field) Marshal() (dAtA []byte, err error) { } func (m *Field) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } - if m.Meta != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n9, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n9 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Views) > 0 { - for _, s := range m.Views { + for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Views[iNdEx]) + copy(dAtA[i:], m.Views[iNdEx]) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Views[iNdEx]))) + i-- dAtA[i] = 0x1a - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Meta != nil { + { + size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *Schema) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2734,32 +2923,40 @@ func (m *Schema) Marshal() (dAtA []byte, err error) { } func (m *Schema) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Schema) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.Indexes) > 0 { - for _, msg := range m.Indexes { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Indexes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Indexes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0xa } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *Index) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2767,38 +2964,47 @@ func (m *Index) Marshal() (dAtA []byte, err error) { } func (m *Index) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Fields[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x22 } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *URI) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2806,37 +3012,45 @@ func (m *URI) Marshal() (dAtA []byte, err error) { } func (m *URI) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *URI) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Scheme) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) - i += copy(dAtA[i:], m.Scheme) - } - if len(m.Host) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) - i += copy(dAtA[i:], m.Host) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Port != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Port)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Host) > 0 { + i -= len(m.Host) + copy(dAtA[i:], m.Host) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Scheme) > 0 { + i -= len(m.Scheme) + copy(dAtA[i:], m.Scheme) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Scheme))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *Node) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2844,52 +3058,62 @@ func (m *Node) Marshal() (dAtA []byte, err error) { } func (m *Node) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Node) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.ID) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) - i += copy(dAtA[i:], m.ID) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.URI != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n10, err := m.URI.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n10 + if len(m.State) > 0 { + i -= len(m.State) + copy(dAtA[i:], m.State) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i-- + dAtA[i] = 0x22 } if m.IsCoordinator { - dAtA[i] = 0x18 - i++ + i-- if m.IsCoordinator { dAtA[i] = 1 } else { dAtA[i] = 0 } - i++ + i-- + dAtA[i] = 0x18 } - if len(m.State) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i += copy(dAtA[i:], m.State) + if m.URI != nil { + { + size, err := m.URI.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.ID) > 0 { + i -= len(m.ID) + copy(dAtA[i:], m.ID) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ID))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2897,32 +3121,40 @@ func (m *NodeStateMessage) Marshal() (dAtA []byte, err error) { } func (m *NodeStateMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeStateMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.NodeID) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) - i += copy(dAtA[i:], m.NodeID) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.State) > 0 { - dAtA[i] = 0x12 - i++ + i -= len(m.State) + copy(dAtA[i:], m.State) i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i += copy(dAtA[i:], m.State) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.NodeID) > 0 { + i -= len(m.NodeID) + copy(dAtA[i:], m.NodeID) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeID))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2930,35 +3162,43 @@ func (m *NodeEventMessage) Marshal() (dAtA []byte, err error) { } func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeEventMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Event != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Node != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n11, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n11 + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Event != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Event)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2966,52 +3206,64 @@ func (m *NodeStatus) Marshal() (dAtA []byte, err error) { } func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Node != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n12, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n12 - } - if m.Schema != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n13, err := m.Schema.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n13 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Indexes) > 0 { - for _, msg := range m.Indexes { + for iNdEx := len(m.Indexes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Indexes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + } + } + if m.Schema != nil { + { + size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Node != nil { + { + size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *IndexStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3019,38 +3271,47 @@ func (m *IndexStatus) Marshal() (dAtA []byte, err error) { } func (m *IndexStatus) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Fields[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x12 } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *FieldStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3058,15 +3319,18 @@ func (m *FieldStatus) Marshal() (dAtA []byte, err error) { } func (m *FieldStatus) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.AvailableShards) > 0 { dAtA15 := make([]byte, len(m.AvailableShards)*10) @@ -3080,21 +3344,26 @@ func (m *FieldStatus) MarshalTo(dAtA []byte) (int, error) { dAtA15[j14] = uint8(num) j14++ } - dAtA[i] = 0x12 - i++ + i -= j14 + copy(dAtA[i:], dAtA15[:j14]) i = encodeVarintPrivate(dAtA, i, uint64(j14)) - i += copy(dAtA[i:], dAtA15[:j14]) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *ClusterStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3102,44 +3371,54 @@ func (m *ClusterStatus) Marshal() (dAtA []byte, err error) { } func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.ClusterID) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) - i += copy(dAtA[i:], m.ClusterID) - } - if len(m.State) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) - i += copy(dAtA[i:], m.State) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Nodes) > 0 { - for _, msg := range m.Nodes { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Nodes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x1a } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.State) > 0 { + i -= len(m.State) + copy(dAtA[i:], m.State) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.ClusterID) > 0 { + i -= len(m.ClusterID) + copy(dAtA[i:], m.ClusterID) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *BSIGroup) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3147,42 +3426,50 @@ func (m *BSIGroup) Marshal() (dAtA []byte, err error) { } func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BSIGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } - if len(m.Type) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) - i += copy(dAtA[i:], m.Type) - } - if m.Min != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Max != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x20 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Min != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + i-- + dAtA[i] = 0x18 } - return i, nil + if len(m.Type) > 0 { + i -= len(m.Type) + copy(dAtA[i:], m.Type) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i-- + dAtA[i] = 0x12 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3190,38 +3477,47 @@ func (m *CreateViewMessage) Marshal() (dAtA []byte, err error) { } func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CreateViewMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.View) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.View) + copy(dAtA[i:], m.View) i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i += copy(dAtA[i:], m.View) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3229,38 +3525,47 @@ func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { } func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DeleteViewMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.View) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.View) + copy(dAtA[i:], m.View) i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i += copy(dAtA[i:], m.View) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3268,77 +3573,93 @@ func (m *ResizeInstruction) Marshal() (dAtA []byte, err error) { } func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ResizeInstruction) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.JobID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Node != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n16, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n16 - } - if m.Coordinator != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n17, err := m.Coordinator.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n17 - } - if len(m.Sources) > 0 { - for _, msg := range m.Sources { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.NodeStatus != nil { + { + size, err := m.NodeStatus.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x3a } if m.ClusterStatus != nil { + { + size, err := m.ClusterStatus.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- dAtA[i] = 0x32 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + } + if len(m.Sources) > 0 { + for iNdEx := len(m.Sources) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Sources[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 } - i += n18 } - if m.NodeStatus != nil { - dAtA[i] = 0x3a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.NodeStatus.Size())) - n19, err := m.NodeStatus.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Coordinator != nil { + { + size, err := m.Coordinator.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) } - i += n19 + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Node != nil { + { + size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 } - return i, nil + if m.JobID != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func (m *ResizeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3346,53 +3667,64 @@ func (m *ResizeSource) Marshal() (dAtA []byte, err error) { } func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ResizeSource) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Node != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n20, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n20 - } - if len(m.Index) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } - if len(m.View) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) - i += copy(dAtA[i:], m.View) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Shard != 0 { - dAtA[i] = 0x28 - i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x28 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.View) > 0 { + i -= len(m.View) + copy(dAtA[i:], m.View) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i-- + dAtA[i] = 0x22 } - return i, nil + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x1a + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0x12 + } + if m.Node != nil { + { + size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3400,41 +3732,50 @@ func (m *ResizeInstructionComplete) Marshal() (dAtA []byte, err error) { } func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ResizeInstructionComplete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.JobID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) - } - if m.Node != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n21, err := m.Node.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n21 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Error) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Error) + copy(dAtA[i:], m.Error) i = encodeVarintPrivate(dAtA, i, uint64(len(m.Error))) - i += copy(dAtA[i:], m.Error) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Node != nil { + { + size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 } - return i, nil + if m.JobID != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.JobID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3442,30 +3783,38 @@ func (m *SetCoordinatorMessage) Marshal() (dAtA []byte, err error) { } func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SetCoordinatorMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.New != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n22 - } if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + if m.New != nil { + { + size, err := m.New.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3473,30 +3822,38 @@ func (m *UpdateCoordinatorMessage) Marshal() (dAtA []byte, err error) { } func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *UpdateCoordinatorMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.New != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n23, err := m.New.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n23 - } if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + if m.New != nil { + { + size, err := m.New.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *Topology) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3504,41 +3861,42 @@ func (m *Topology) Marshal() (dAtA []byte, err error) { } func (m *Topology) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Topology) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.ClusterID) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) - i += copy(dAtA[i:], m.ClusterID) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.NodeIDs) > 0 { - for _, s := range m.NodeIDs { + for iNdEx := len(m.NodeIDs) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.NodeIDs[iNdEx]) + copy(dAtA[i:], m.NodeIDs[iNdEx]) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.NodeIDs[iNdEx]))) + i-- dAtA[i] = 0x12 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.ClusterID) > 0 { + i -= len(m.ClusterID) + copy(dAtA[i:], m.ClusterID) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClusterID))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -3546,24 +3904,32 @@ func (m *RecalculateCaches) Marshal() (dAtA []byte, err error) { } func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RecalculateCaches) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + return len(dAtA) - i, nil } func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { + offset -= sovPrivate(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func (m *IndexMeta) Size() (n int) { if m == nil { @@ -4352,14 +4718,7 @@ func (m *RecalculateCaches) Size() (n int) { } func sovPrivate(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozPrivate(x uint64) (n int) { return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -4379,7 +4738,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4407,7 +4766,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4427,7 +4786,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4442,6 +4801,9 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4470,7 +4832,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4498,7 +4860,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4508,6 +4870,9 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4527,7 +4892,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.CacheSize |= (uint32(b) & 0x7F) << shift + m.CacheSize |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -4546,7 +4911,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4556,6 +4921,9 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4575,7 +4943,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4585,6 +4953,9 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4604,7 +4975,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Min |= (int64(b) & 0x7F) << shift + m.Min |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4623,7 +4994,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Max |= (int64(b) & 0x7F) << shift + m.Max |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4642,7 +5013,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4662,7 +5033,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4682,7 +5053,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Base |= (int64(b) & 0x7F) << shift + m.Base |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4701,7 +5072,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.BitDepth |= (uint64(b) & 0x7F) << shift + m.BitDepth |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4720,7 +5091,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Scale |= (int64(b) & 0x7F) << shift + m.Scale |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4734,6 +5105,9 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4762,7 +5136,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4790,7 +5164,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4800,6 +5174,9 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4814,6 +5191,9 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4842,7 +5222,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4870,7 +5250,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4880,6 +5260,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4899,7 +5282,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4909,6 +5292,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4928,7 +5314,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Block |= (uint64(b) & 0x7F) << shift + m.Block |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4947,7 +5333,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (uint64(b) & 0x7F) << shift + m.Shard |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4966,7 +5352,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4976,6 +5362,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4990,6 +5379,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5018,7 +5410,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5044,7 +5436,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5061,7 +5453,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5070,12 +5462,15 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -5095,7 +5490,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5117,7 +5512,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5134,7 +5529,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5143,12 +5538,15 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -5168,7 +5566,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5187,6 +5585,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5215,7 +5616,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5241,7 +5642,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5258,7 +5659,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5267,12 +5668,15 @@ func (m *Cache) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -5292,7 +5696,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5311,6 +5715,9 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5339,7 +5746,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5367,7 +5774,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5376,6 +5783,9 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5396,7 +5806,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5413,7 +5823,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift + stringLenmapkey |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5423,6 +5833,9 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPrivate + } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -5438,7 +5851,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift + mapvalue |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5469,6 +5882,9 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5497,7 +5913,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5525,7 +5941,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5535,6 +5951,9 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5554,7 +5973,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (uint64(b) & 0x7F) << shift + m.Shard |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5573,7 +5992,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5583,6 +6002,9 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5597,6 +6019,9 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5625,7 +6050,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5653,7 +6078,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5663,6 +6088,9 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5677,6 +6105,9 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5705,7 +6136,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5733,7 +6164,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5743,6 +6174,9 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5762,7 +6196,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5771,6 +6205,9 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5790,6 +6227,9 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5818,7 +6258,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5846,7 +6286,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5856,6 +6296,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5875,7 +6318,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5885,6 +6328,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5904,7 +6350,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5913,6 +6359,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5932,6 +6381,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5960,7 +6412,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5988,7 +6440,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5998,6 +6450,9 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6017,7 +6472,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6027,6 +6482,9 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6041,6 +6499,9 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6069,7 +6530,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6097,7 +6558,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6107,6 +6568,9 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6126,7 +6590,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6136,6 +6600,9 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6155,7 +6622,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ShardID |= (uint64(b) & 0x7F) << shift + m.ShardID |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6169,6 +6636,9 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6197,7 +6667,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6225,7 +6695,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6235,6 +6705,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6254,7 +6727,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6263,6 +6736,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6287,7 +6763,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6297,6 +6773,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6311,6 +6790,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6339,7 +6821,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6367,7 +6849,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6376,6 +6858,9 @@ func (m *Schema) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6393,6 +6878,9 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6421,7 +6909,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6449,7 +6937,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6459,6 +6947,9 @@ func (m *Index) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6478,7 +6969,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6487,6 +6978,9 @@ func (m *Index) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6504,6 +6998,9 @@ func (m *Index) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6532,7 +7029,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6560,7 +7057,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6570,6 +7067,9 @@ func (m *URI) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6589,7 +7089,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6599,6 +7099,9 @@ func (m *URI) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6618,7 +7121,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Port |= (uint32(b) & 0x7F) << shift + m.Port |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -6632,6 +7135,9 @@ func (m *URI) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6660,7 +7166,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6688,7 +7194,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6698,6 +7204,9 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6717,7 +7226,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6726,6 +7235,9 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6750,7 +7262,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6770,7 +7282,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6780,6 +7292,9 @@ func (m *Node) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6794,6 +7309,9 @@ func (m *Node) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6822,7 +7340,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6850,7 +7368,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6860,6 +7378,9 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6879,7 +7400,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6889,6 +7410,9 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6903,6 +7427,9 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6931,7 +7458,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6959,7 +7486,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Event |= (uint32(b) & 0x7F) << shift + m.Event |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -6978,7 +7505,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6987,6 +7514,9 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7006,6 +7536,9 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7034,7 +7567,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7062,7 +7595,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7071,6 +7604,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7095,7 +7631,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7104,6 +7640,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7128,7 +7667,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7137,6 +7676,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7154,6 +7696,9 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7182,7 +7727,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7210,7 +7755,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7220,6 +7765,9 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7239,7 +7787,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7248,6 +7796,9 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7265,6 +7816,9 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7293,7 +7847,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7321,7 +7875,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7331,6 +7885,9 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7348,7 +7905,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7365,7 +7922,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7374,12 +7931,15 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -7399,7 +7959,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7418,6 +7978,9 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7446,7 +8009,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7474,7 +8037,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7484,6 +8047,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7503,7 +8069,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7513,6 +8079,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7532,7 +8101,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7541,6 +8110,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7558,6 +8130,9 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7586,7 +8161,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7614,7 +8189,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7624,6 +8199,9 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7643,7 +8221,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7653,6 +8231,9 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7672,7 +8253,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Min |= (int64(b) & 0x7F) << shift + m.Min |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7691,7 +8272,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Max |= (int64(b) & 0x7F) << shift + m.Max |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7705,6 +8286,9 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7733,7 +8317,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7761,7 +8345,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7771,6 +8355,9 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7790,7 +8377,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7800,6 +8387,9 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7819,7 +8409,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7829,6 +8419,9 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7843,6 +8436,9 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7871,7 +8467,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7899,7 +8495,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7909,6 +8505,9 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7928,7 +8527,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7938,6 +8537,9 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7957,7 +8559,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7967,6 +8569,9 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7981,6 +8586,9 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8009,7 +8617,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8037,7 +8645,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.JobID |= (int64(b) & 0x7F) << shift + m.JobID |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8056,7 +8664,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8065,6 +8673,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8089,7 +8700,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8098,6 +8709,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8122,7 +8736,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8131,6 +8745,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8153,7 +8770,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8162,6 +8779,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8186,7 +8806,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8195,6 +8815,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8214,6 +8837,9 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8242,7 +8868,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8270,7 +8896,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8279,6 +8905,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8303,7 +8932,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8313,6 +8942,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8332,7 +8964,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8342,6 +8974,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8361,7 +8996,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8371,6 +9006,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8390,7 +9028,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (uint64(b) & 0x7F) << shift + m.Shard |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8404,6 +9042,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8432,7 +9073,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8460,7 +9101,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.JobID |= (int64(b) & 0x7F) << shift + m.JobID |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8479,7 +9120,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8488,6 +9129,9 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8512,7 +9156,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8522,6 +9166,9 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8536,6 +9183,9 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8564,7 +9214,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8592,7 +9242,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8601,6 +9251,9 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8620,6 +9273,9 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8648,7 +9304,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8676,7 +9332,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -8685,6 +9341,9 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8704,6 +9363,9 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8732,7 +9394,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8760,7 +9422,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8770,6 +9432,9 @@ func (m *Topology) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8789,7 +9454,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8799,6 +9464,9 @@ func (m *Topology) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPrivate } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -8813,6 +9481,9 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8841,7 +9512,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -8864,6 +9535,9 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPrivate } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -8880,6 +9554,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -8911,10 +9586,8 @@ func skipPrivate(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -8931,133 +9604,34 @@ func skipPrivate(dAtA []byte) (n int, err error) { break } } - iNdEx += length if length < 0 { return 0, ErrInvalidLengthPrivate } - return iNdEx, nil + iNdEx += length case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPrivate - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipPrivate(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupPrivate + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthPrivate + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthPrivate = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthPrivate = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupPrivate = fmt.Errorf("proto: unexpected end of group") ) - -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_e6d12fddb5948a73) } - -var fileDescriptor_private_e6d12fddb5948a73 = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, - 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x71, 0x0e, 0xdb, 0x36, 0x6c, 0x0b, 0x0a, 0x66, 0x54, 0x51, - 0x53, 0x89, 0x50, 0xb5, 0x5c, 0x70, 0xaa, 0x54, 0x1c, 0x87, 0xb2, 0x94, 0x84, 0x32, 0x4e, 0x72, - 0xc7, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90, - 0x78, 0x01, 0x9e, 0xa0, 0xcf, 0xc2, 0x25, 0x8f, 0x80, 0xc2, 0x8b, 0xa0, 0xf9, 0x67, 0xf6, 0x60, - 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xfe, 0xd3, 0xf7, 0x9f, 0x7f, 0xaf, 0xa1, 0x35, 0x4e, 0xc2, 0x13, - 0x26, 0xf9, 0xe6, 0x38, 0x11, 0x52, 0x78, 0xf5, 0x30, 0x96, 0x3c, 0x89, 0x59, 0x44, 0x9e, 0x43, - 0x23, 0x88, 0x87, 0xfc, 0x6c, 0x87, 0x4b, 0xe6, 0x79, 0xe0, 0xbe, 0xe0, 0x93, 0xd4, 0x77, 0xda, - 0x56, 0xa7, 0x4e, 0xf1, 0xed, 0x7d, 0x00, 0xcb, 0x7b, 0x09, 0x1b, 0x1c, 0x6f, 0x9f, 0x85, 0xa9, - 0xe4, 0xf1, 0x80, 0xfb, 0x2e, 0x4a, 0x67, 0xb8, 0xe4, 0x8d, 0x0d, 0x4b, 0x5f, 0x87, 0x3c, 0x1a, - 0x7e, 0x3f, 0x96, 0xa1, 0x88, 0x53, 0xef, 0x5d, 0x68, 0x6c, 0xb1, 0xc1, 0x11, 0xdf, 0x9b, 0x8c, - 0x39, 0x22, 0x36, 0x68, 0xc9, 0x28, 0xa4, 0xfd, 0xf0, 0xb5, 0x46, 0x6c, 0xd1, 0x92, 0xe1, 0xb5, - 0xa1, 0xb9, 0x17, 0x8e, 0xf8, 0x0f, 0x19, 0x8b, 0x65, 0x36, 0xf2, 0x17, 0xd0, 0xba, 0xca, 0x52, - 0xa1, 0x22, 0x70, 0x1d, 0x45, 0xf8, 0xf6, 0x56, 0xc1, 0xd9, 0x09, 0x63, 0xbf, 0xd1, 0xb6, 0x3a, - 0x0e, 0x55, 0x4f, 0xe4, 0xb0, 0x33, 0x1f, 0x0c, 0x87, 0x9d, 0x15, 0x29, 0x36, 0xa7, 0x53, 0xdc, - 0x15, 0x7d, 0xc9, 0xe2, 0x21, 0x4b, 0x86, 0x07, 0x21, 0x3f, 0xf5, 0x97, 0x74, 0x8a, 0xd3, 0x5c, - 0x65, 0xdb, 0x65, 0x29, 0xf7, 0x5b, 0x08, 0x87, 0x6f, 0xef, 0x1e, 0xd4, 0xbb, 0xa1, 0xec, 0xf1, - 0xb1, 0x3c, 0xf2, 0x97, 0xdb, 0x56, 0xc7, 0xa5, 0x05, 0xed, 0xdd, 0x86, 0x85, 0xfe, 0x80, 0x45, - 0xdc, 0x5f, 0x41, 0x03, 0x4d, 0x10, 0x02, 0xcb, 0xc1, 0x68, 0x2c, 0x12, 0x49, 0x79, 0x3a, 0x16, - 0x71, 0x8a, 0x71, 0x6f, 0x27, 0x89, 0x6f, 0x61, 0x2a, 0xea, 0x49, 0x7e, 0x86, 0xd5, 0x6e, 0x24, - 0x06, 0xc7, 0x3d, 0x26, 0x19, 0xe5, 0x3f, 0x65, 0x3c, 0x95, 0x0a, 0x0d, 0x3b, 0x65, 0xf4, 0x34, - 0xa1, 0xb8, 0x58, 0x75, 0xdf, 0xd6, 0x5c, 0x24, 0x14, 0x17, 0xed, 0xb1, 0xee, 0x2e, 0xd5, 0x04, - 0xc6, 0x73, 0xc4, 0x92, 0x21, 0xd6, 0xdb, 0xa5, 0x9a, 0x50, 0x59, 0x61, 0xce, 0xba, 0xc8, 0xf8, - 0x26, 0x01, 0xac, 0x55, 0xfc, 0x9b, 0x30, 0xd7, 0xa1, 0x46, 0xc5, 0x69, 0xd0, 0x4b, 0x7d, 0xab, - 0xed, 0x74, 0x5c, 0x6a, 0x28, 0x6c, 0xa5, 0x88, 0xb2, 0x51, 0xac, 0x44, 0x36, 0x8a, 0x4a, 0x06, - 0xb9, 0x0b, 0x0b, 0xd8, 0x57, 0x95, 0x65, 0x69, 0xab, 0x9e, 0xe4, 0x17, 0x0b, 0x1a, 0x3b, 0xec, - 0x0c, 0xc3, 0x48, 0xbd, 0xa7, 0x50, 0xcf, 0xab, 0x8d, 0x4a, 0xcd, 0xc7, 0xef, 0x6f, 0xe6, 0x63, - 0xba, 0x59, 0xa8, 0x6d, 0xe6, 0x3a, 0xdb, 0xb1, 0x4c, 0x26, 0xb4, 0x30, 0xb9, 0xf7, 0x05, 0xb4, - 0xa6, 0x44, 0xca, 0xdf, 0x31, 0x9f, 0xe4, 0x55, 0x3d, 0xe6, 0x13, 0x95, 0xff, 0x09, 0x8b, 0x32, - 0x8e, 0xb5, 0x72, 0xa9, 0x26, 0x3e, 0xb7, 0x3f, 0xb5, 0xc8, 0x01, 0x78, 0x5b, 0x09, 0x67, 0x92, - 0xa3, 0x93, 0x1d, 0x9e, 0xa6, 0xec, 0x15, 0xbf, 0xbc, 0xe2, 0xba, 0x8a, 0x76, 0xb5, 0x8a, 0x45, - 0x1f, 0x9c, 0x4a, 0x1f, 0xc8, 0x43, 0xf0, 0x7a, 0x3c, 0xe2, 0x92, 0x9b, 0x1d, 0xfb, 0x17, 0x5c, - 0xd2, 0xcf, 0x63, 0xb8, 0x5a, 0xd7, 0x7b, 0x00, 0xae, 0x5a, 0x58, 0x0c, 0xa1, 0xf9, 0xf8, 0x56, - 0x59, 0xa7, 0x62, 0x97, 0x29, 0x2a, 0x90, 0x28, 0x07, 0xc5, 0x78, 0xae, 0x4c, 0x6c, 0xce, 0x28, - 0x3d, 0x34, 0xae, 0x1c, 0x74, 0xb5, 0x5e, 0xba, 0xaa, 0x2e, 0xbb, 0xf1, 0xf6, 0x2c, 0x4f, 0xf7, - 0xa6, 0xde, 0xc8, 0x00, 0xde, 0xd1, 0x08, 0x5f, 0x9d, 0xb0, 0x30, 0x62, 0x87, 0xd1, 0x35, 0x3b, - 0x32, 0x27, 0x70, 0x1f, 0x16, 0xd1, 0x36, 0xe8, 0x99, 0x2d, 0xc8, 0x49, 0xf2, 0xa3, 0xd1, 0x57, - 0xa3, 0xbf, 0xcb, 0x46, 0xdc, 0xa0, 0xe1, 0xbb, 0xc8, 0xd7, 0xbe, 0x3a, 0x5f, 0xe5, 0x58, 0xad, - 0x8b, 0x3a, 0x98, 0x8e, 0x72, 0x8c, 0x04, 0x79, 0x02, 0xb5, 0xfe, 0xe0, 0x88, 0x8f, 0x98, 0xf7, - 0x21, 0x2c, 0x62, 0x84, 0x3c, 0x35, 0x13, 0xbd, 0x32, 0xd3, 0x29, 0x9a, 0xcb, 0x49, 0xcf, 0x64, - 0x36, 0x37, 0xa6, 0x07, 0x50, 0x43, 0xef, 0xa9, 0xef, 0xce, 0xc2, 0x20, 0x9f, 0x1a, 0x31, 0xd9, - 0x06, 0x67, 0x9f, 0x06, 0x6a, 0x53, 0x31, 0x82, 0x1c, 0xc5, 0x50, 0x0a, 0xfb, 0x1b, 0x91, 0x4a, - 0x53, 0x27, 0x7c, 0x2b, 0xde, 0x4b, 0x91, 0x48, 0xac, 0x51, 0x8b, 0xe2, 0x9b, 0xa4, 0xe0, 0xee, - 0x8a, 0x21, 0xf7, 0x96, 0xc1, 0x0e, 0x7a, 0x06, 0xc3, 0x0e, 0x7a, 0xde, 0x7b, 0x08, 0x6f, 0x4a, - 0xd3, 0x2a, 0x83, 0xd8, 0xa7, 0x01, 0x45, 0xc7, 0xf7, 0xa1, 0x15, 0xa4, 0x5b, 0x42, 0x24, 0xc3, - 0x30, 0x66, 0x52, 0x24, 0xe6, 0x97, 0x64, 0x9a, 0x89, 0x1b, 0x24, 0x99, 0xd4, 0x77, 0xbf, 0x41, - 0x35, 0x41, 0x9e, 0xc1, 0xaa, 0x72, 0x8a, 0x44, 0xde, 0xef, 0x75, 0xa8, 0x29, 0x5e, 0x11, 0x84, - 0xa1, 0x4a, 0x04, 0xbb, 0x8a, 0xf0, 0x9d, 0x46, 0xd8, 0x3e, 0xe1, 0xb1, 0xac, 0x4c, 0x0c, 0xd2, - 0x08, 0xd0, 0xa2, 0x9a, 0xf0, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x5c, 0x66, 0xa2, 0xb8, 0x14, 0x65, - 0xe4, 0x37, 0x0b, 0x20, 0x0f, 0x28, 0x4b, 0x0b, 0x13, 0xeb, 0x72, 0x13, 0xaf, 0x93, 0x77, 0xde, - 0x6c, 0xcb, 0x6a, 0xa9, 0xa5, 0xf9, 0x34, 0x9f, 0x8c, 0x8f, 0xcb, 0xc9, 0xd0, 0x2d, 0xbd, 0x33, - 0x33, 0x19, 0xda, 0x6b, 0x39, 0x1f, 0x2f, 0xa1, 0x59, 0xe1, 0xcf, 0x9d, 0x92, 0x8f, 0x8a, 0x29, - 0xb1, 0x67, 0x21, 0x91, 0x6f, 0x20, 0xf3, 0x59, 0x79, 0x01, 0xcd, 0x0a, 0x7b, 0x2e, 0x62, 0x07, - 0x56, 0xa6, 0xf7, 0x30, 0xbf, 0xef, 0xb3, 0x6c, 0x12, 0x42, 0x6b, 0x2b, 0xca, 0x52, 0xc9, 0x13, - 0x03, 0xa7, 0x7e, 0x14, 0x34, 0xa3, 0x68, 0x5e, 0xc9, 0x98, 0xdf, 0x3f, 0xef, 0x3e, 0x2c, 0xa8, - 0x32, 0xea, 0x75, 0xba, 0x58, 0x63, 0x2d, 0x24, 0x07, 0x50, 0xef, 0xf6, 0x83, 0xe7, 0x89, 0xc8, - 0xc6, 0x73, 0x83, 0xce, 0xbf, 0x0c, 0xec, 0x8b, 0x5f, 0x06, 0xce, 0x85, 0x2f, 0x03, 0xb7, 0xf8, - 0x32, 0x20, 0x7d, 0x58, 0xd3, 0xa7, 0x52, 0x6d, 0xf1, 0x4d, 0x0e, 0x4e, 0xfe, 0x43, 0xea, 0x54, - 0x7e, 0x48, 0xfb, 0xb0, 0xa6, 0xef, 0xd9, 0xff, 0x09, 0xfa, 0xc6, 0x86, 0x35, 0xca, 0xd3, 0xf0, - 0x35, 0x0f, 0xe2, 0x54, 0x26, 0xd9, 0x40, 0xdd, 0x24, 0x65, 0xff, 0xad, 0x38, 0x34, 0xd5, 0x76, - 0xa8, 0x26, 0xae, 0x33, 0xe9, 0xde, 0x23, 0x68, 0xce, 0xee, 0xec, 0x45, 0xd5, 0xaa, 0x8a, 0xf7, - 0x08, 0x16, 0xfb, 0x22, 0x4b, 0x06, 0xc5, 0xf8, 0x56, 0xee, 0xa4, 0x8e, 0x4c, 0x8b, 0x69, 0xae, - 0xe6, 0x3d, 0x9d, 0x19, 0x10, 0xbf, 0x86, 0x5e, 0xde, 0x2e, 0xed, 0xa6, 0xc4, 0x74, 0x66, 0x9c, - 0x3e, 0xa9, 0xee, 0xa2, 0xbf, 0x88, 0xb6, 0xb7, 0xa7, 0x23, 0x34, 0x86, 0x15, 0x3d, 0xf2, 0xab, - 0x05, 0x4b, 0xd5, 0x70, 0xae, 0xb5, 0xc4, 0x45, 0x77, 0xec, 0xb9, 0xdd, 0x71, 0xe6, 0x75, 0xc7, - 0x2d, 0xbb, 0x53, 0x7e, 0x1f, 0x2c, 0x54, 0xbe, 0x0f, 0xc8, 0x31, 0xdc, 0xbd, 0xd0, 0xb2, 0x2d, - 0x31, 0x1a, 0xab, 0xd9, 0xf8, 0x0f, 0xad, 0x53, 0xe7, 0x2d, 0x49, 0x4c, 0xd3, 0x1a, 0x54, 0x13, - 0xe4, 0x33, 0xb8, 0xd3, 0xe7, 0xb2, 0xd2, 0xb0, 0x7c, 0xf2, 0xda, 0xe0, 0xec, 0xf2, 0xd3, 0x4b, - 0xd2, 0x57, 0x22, 0xf2, 0x25, 0xf8, 0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xd6, 0x5d, 0xa8, 0xef, - 0x89, 0xb1, 0x88, 0xc4, 0xab, 0xc9, 0x15, 0x17, 0xc0, 0x87, 0x45, 0x7d, 0xcb, 0xf5, 0x49, 0x69, - 0xd0, 0x9c, 0x24, 0xb7, 0xd4, 0x70, 0x0f, 0x58, 0x34, 0xc8, 0x22, 0x15, 0x86, 0xfa, 0x76, 0x4c, - 0xbb, 0xab, 0x7f, 0x9c, 0x6f, 0x58, 0x7f, 0x9e, 0x6f, 0x58, 0x7f, 0x9d, 0x6f, 0x58, 0xbf, 0xff, - 0xbd, 0xf1, 0xd6, 0x61, 0x0d, 0xff, 0xc9, 0x3c, 0xf9, 0x27, 0x00, 0x00, 0xff, 0xff, 0xb4, 0x70, - 0x9a, 0xfe, 0xda, 0x0c, 0x00, 0x00, -} diff --git a/internal/public.pb.go b/internal/public.pb.go index 08ed42318..dfae6c59c 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -3,13 +3,14 @@ package internal -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -import encoding_binary "encoding/binary" - -import io "io" +import ( + encoding_binary "encoding/binary" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -20,12 +21,12 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Row struct { - Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns,proto3" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"` Roaring []byte `protobuf:"bytes,4,opt,name=Roaring,proto3" json:"Roaring,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -36,7 +37,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{0} + return fileDescriptor_413a91106d7bcce8, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -46,15 +47,15 @@ func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Row.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(dst, src) +func (m *Row) XXX_Merge(src proto.Message) { + xxx_messageInfo_Row.Merge(m, src) } func (m *Row) XXX_Size() int { return m.Size() @@ -94,8 +95,8 @@ func (m *Row) GetRoaring() []byte { } type SignedRow struct { - Pos *Row `protobuf:"bytes,1,opt,name=Pos" json:"Pos,omitempty"` - Neg *Row `protobuf:"bytes,2,opt,name=Neg" json:"Neg,omitempty"` + Pos *Row `protobuf:"bytes,1,opt,name=Pos,proto3" json:"Pos,omitempty"` + Neg *Row `protobuf:"bytes,2,opt,name=Neg,proto3" json:"Neg,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -105,7 +106,7 @@ func (m *SignedRow) Reset() { *m = SignedRow{} } func (m *SignedRow) String() string { return proto.CompactTextString(m) } func (*SignedRow) ProtoMessage() {} func (*SignedRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{1} + return fileDescriptor_413a91106d7bcce8, []int{1} } func (m *SignedRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -115,15 +116,15 @@ func (m *SignedRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_SignedRow.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *SignedRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_SignedRow.Merge(dst, src) +func (m *SignedRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_SignedRow.Merge(m, src) } func (m *SignedRow) XXX_Size() int { return m.Size() @@ -149,8 +150,8 @@ func (m *SignedRow) GetNeg() *Row { } type RowIdentifiers struct { - Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows,proto3" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys,proto3" json:"Keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -160,7 +161,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{2} + return fileDescriptor_413a91106d7bcce8, []int{2} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -170,15 +171,15 @@ func (m *RowIdentifiers) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_RowIdentifiers.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *RowIdentifiers) XXX_Merge(src proto.Message) { - xxx_messageInfo_RowIdentifiers.Merge(dst, src) +func (m *RowIdentifiers) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowIdentifiers.Merge(m, src) } func (m *RowIdentifiers) XXX_Size() int { return m.Size() @@ -216,7 +217,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{3} + return fileDescriptor_413a91106d7bcce8, []int{3} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -226,15 +227,15 @@ func (m *Pair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Pair.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Pair) XXX_Merge(src proto.Message) { - xxx_messageInfo_Pair.Merge(dst, src) +func (m *Pair) XXX_Merge(src proto.Message) { + xxx_messageInfo_Pair.Merge(m, src) } func (m *Pair) XXX_Size() int { return m.Size() @@ -267,7 +268,7 @@ func (m *Pair) GetCount() uint64 { } type PairField struct { - Pair *Pair `protobuf:"bytes,1,opt,name=Pair" json:"Pair,omitempty"` + Pair *Pair `protobuf:"bytes,1,opt,name=Pair,proto3" json:"Pair,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -278,7 +279,7 @@ func (m *PairField) Reset() { *m = PairField{} } func (m *PairField) String() string { return proto.CompactTextString(m) } func (*PairField) ProtoMessage() {} func (*PairField) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{4} + return fileDescriptor_413a91106d7bcce8, []int{4} } func (m *PairField) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -288,15 +289,15 @@ func (m *PairField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PairField.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *PairField) XXX_Merge(src proto.Message) { - xxx_messageInfo_PairField.Merge(dst, src) +func (m *PairField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairField.Merge(m, src) } func (m *PairField) XXX_Size() int { return m.Size() @@ -322,7 +323,7 @@ func (m *PairField) GetField() string { } type PairsField struct { - Pairs []*Pair `protobuf:"bytes,1,rep,name=Pairs" json:"Pairs,omitempty"` + Pairs []*Pair `protobuf:"bytes,1,rep,name=Pairs,proto3" json:"Pairs,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -333,7 +334,7 @@ func (m *PairsField) Reset() { *m = PairsField{} } func (m *PairsField) String() string { return proto.CompactTextString(m) } func (*PairsField) ProtoMessage() {} func (*PairsField) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{5} + return fileDescriptor_413a91106d7bcce8, []int{5} } func (m *PairsField) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -343,15 +344,15 @@ func (m *PairsField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PairsField.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *PairsField) XXX_Merge(src proto.Message) { - xxx_messageInfo_PairsField.Merge(dst, src) +func (m *PairsField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairsField.Merge(m, src) } func (m *PairsField) XXX_Size() int { return m.Size() @@ -389,7 +390,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{6} + return fileDescriptor_413a91106d7bcce8, []int{6} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -399,15 +400,15 @@ func (m *FieldRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FieldRow.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *FieldRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FieldRow.Merge(dst, src) +func (m *FieldRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldRow.Merge(m, src) } func (m *FieldRow) XXX_Size() int { return m.Size() @@ -440,7 +441,7 @@ func (m *FieldRow) GetRowKey() string { } type GroupCount struct { - Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group,proto3" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` Sum int64 `protobuf:"varint,3,opt,name=Sum,proto3" json:"Sum,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -452,7 +453,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{7} + return fileDescriptor_413a91106d7bcce8, []int{7} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -462,15 +463,15 @@ func (m *GroupCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GroupCount.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *GroupCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_GroupCount.Merge(dst, src) +func (m *GroupCount) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupCount.Merge(m, src) } func (m *GroupCount) XXX_Size() int { return m.Size() @@ -514,7 +515,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{8} + return fileDescriptor_413a91106d7bcce8, []int{8} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -524,15 +525,15 @@ func (m *ValCount) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValCount.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ValCount) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValCount.Merge(dst, src) +func (m *ValCount) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValCount.Merge(m, src) } func (m *ValCount) XXX_Size() int { return m.Size() @@ -560,7 +561,7 @@ func (m *ValCount) GetCount() int64 { type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -570,7 +571,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{9} + return fileDescriptor_413a91106d7bcce8, []int{9} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -580,15 +581,15 @@ func (m *ColumnAttrSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ColumnAttrSet.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ColumnAttrSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnAttrSet.Merge(dst, src) +func (m *ColumnAttrSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnAttrSet.Merge(m, src) } func (m *ColumnAttrSet) XXX_Size() int { return m.Size() @@ -636,7 +637,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{10} + return fileDescriptor_413a91106d7bcce8, []int{10} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -646,15 +647,15 @@ func (m *Attr) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attr.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Attr) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attr.Merge(dst, src) +func (m *Attr) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attr.Merge(m, src) } func (m *Attr) XXX_Size() int { return m.Size() @@ -708,7 +709,7 @@ func (m *Attr) GetFloatValue() float64 { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,proto3" json:"Attrs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -718,7 +719,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{11} + return fileDescriptor_413a91106d7bcce8, []int{11} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,15 +729,15 @@ func (m *AttrMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_AttrMap.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *AttrMap) XXX_Merge(src proto.Message) { - xxx_messageInfo_AttrMap.Merge(dst, src) +func (m *AttrMap) XXX_Merge(src proto.Message) { + xxx_messageInfo_AttrMap.Merge(m, src) } func (m *AttrMap) XXX_Size() int { return m.Size() @@ -756,12 +757,12 @@ func (m *AttrMap) GetAttrs() []*Attr { type QueryRequest struct { Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards,proto3" json:"Shards,omitempty"` ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` - EmbeddedData []*Row `protobuf:"bytes,8,rep,name=EmbeddedData" json:"EmbeddedData,omitempty"` + EmbeddedData []*Row `protobuf:"bytes,8,rep,name=EmbeddedData,proto3" json:"EmbeddedData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -771,7 +772,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{12} + return fileDescriptor_413a91106d7bcce8, []int{12} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -781,15 +782,15 @@ func (m *QueryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_QueryRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *QueryRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryRequest.Merge(dst, src) +func (m *QueryRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryRequest.Merge(m, src) } func (m *QueryRequest) XXX_Size() int { return m.Size() @@ -851,8 +852,8 @@ func (m *QueryRequest) GetEmbeddedData() []*Row { type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,proto3" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets,proto3" json:"ColumnAttrSets,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -862,7 +863,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{13} + return fileDescriptor_413a91106d7bcce8, []int{13} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -872,15 +873,15 @@ func (m *QueryResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_QueryResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *QueryResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResponse.Merge(dst, src) +func (m *QueryResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResponse.Merge(m, src) } func (m *QueryResponse) XXX_Size() int { return m.Size() @@ -914,16 +915,16 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { type QueryResult struct { Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` - RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` - SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow" json:"SignedRow,omitempty"` - PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField" json:"PairsField,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers,proto3" json:"RowIdentifiers,omitempty"` + SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow,proto3" json:"SignedRow,omitempty"` + PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField,proto3" json:"PairsField,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -933,7 +934,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{14} + return fileDescriptor_413a91106d7bcce8, []int{14} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -943,15 +944,15 @@ func (m *QueryResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_QueryResult.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *QueryResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResult.Merge(dst, src) +func (m *QueryResult) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResult.Merge(m, src) } func (m *QueryResult) XXX_Size() int { return m.Size() @@ -1043,11 +1044,11 @@ type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` - ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` + ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1057,7 +1058,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{15} + return fileDescriptor_413a91106d7bcce8, []int{15} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1067,15 +1068,15 @@ func (m *ImportRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return xxx_messageInfo_ImportRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRequest.Merge(dst, src) +func (m *ImportRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRequest.Merge(m, src) } func (m *ImportRequest) XXX_Size() int { return m.Size() @@ -1146,10 +1147,10 @@ type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` - Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` - FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues" json:"FloatValues,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` + FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1159,7 +1160,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{16} + return fileDescriptor_413a91106d7bcce8, []int{16} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1169,15 +1170,15 @@ func (m *ImportValueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, return xxx_messageInfo_ImportValueRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportValueRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportValueRequest.Merge(dst, src) +func (m *ImportValueRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportValueRequest.Merge(m, src) } func (m *ImportValueRequest) XXX_Size() int { return m.Size() @@ -1240,7 +1241,7 @@ func (m *ImportValueRequest) GetFloatValues() []float64 { type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1250,7 +1251,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{17} + return fileDescriptor_413a91106d7bcce8, []int{17} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1260,15 +1261,15 @@ func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) +func (m *TranslateKeysRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysRequest.Merge(m, src) } func (m *TranslateKeysRequest) XXX_Size() int { return m.Size() @@ -1301,7 +1302,7 @@ func (m *TranslateKeysRequest) GetKeys() []string { } type TranslateKeysResponse struct { - IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs,proto3" json:"IDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1311,7 +1312,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{18} + return fileDescriptor_413a91106d7bcce8, []int{18} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1321,15 +1322,15 @@ func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byt return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) +func (m *TranslateKeysResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysResponse.Merge(m, src) } func (m *TranslateKeysResponse) XXX_Size() int { return m.Size() @@ -1347,6 +1348,116 @@ func (m *TranslateKeysResponse) GetIDs() []uint64 { return nil } +type TranslateIDsRequest struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs,proto3" json:"IDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateIDsRequest) Reset() { *m = TranslateIDsRequest{} } +func (m *TranslateIDsRequest) String() string { return proto.CompactTextString(m) } +func (*TranslateIDsRequest) ProtoMessage() {} +func (*TranslateIDsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{19} +} +func (m *TranslateIDsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateIDsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateIDsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *TranslateIDsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateIDsRequest.Merge(m, src) +} +func (m *TranslateIDsRequest) XXX_Size() int { + return m.Size() +} +func (m *TranslateIDsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateIDsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateIDsRequest proto.InternalMessageInfo + +func (m *TranslateIDsRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *TranslateIDsRequest) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *TranslateIDsRequest) GetIDs() []uint64 { + if m != nil { + return m.IDs + } + return nil +} + +type TranslateIDsResponse struct { + Keys []string `protobuf:"bytes,3,rep,name=Keys,proto3" json:"Keys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateIDsResponse) Reset() { *m = TranslateIDsResponse{} } +func (m *TranslateIDsResponse) String() string { return proto.CompactTextString(m) } +func (*TranslateIDsResponse) ProtoMessage() {} +func (*TranslateIDsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{20} +} +func (m *TranslateIDsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateIDsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateIDsResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *TranslateIDsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateIDsResponse.Merge(m, src) +} +func (m *TranslateIDsResponse) XXX_Size() int { + return m.Size() +} +func (m *TranslateIDsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateIDsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateIDsResponse proto.InternalMessageInfo + +func (m *TranslateIDsResponse) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + type ImportRoaringRequestView struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` @@ -1359,7 +1470,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{19} + return fileDescriptor_413a91106d7bcce8, []int{21} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1369,15 +1480,15 @@ func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) +func (m *ImportRoaringRequestView) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequestView.Merge(m, src) } func (m *ImportRoaringRequestView) XXX_Size() int { return m.Size() @@ -1404,7 +1515,7 @@ func (m *ImportRoaringRequestView) GetData() []byte { type ImportRoaringRequest struct { Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` - Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1414,7 +1525,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{20} + return fileDescriptor_413a91106d7bcce8, []int{22} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1424,15 +1535,15 @@ func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) +func (m *ImportRoaringRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequest.Merge(m, src) } func (m *ImportRoaringRequest) XXX_Size() int { return m.Size() @@ -1461,8 +1572,8 @@ type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"` - AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals" json:"AttrVals,omitempty"` - ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` + AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1472,7 +1583,7 @@ func (m *ImportColumnAttrsRequest) Reset() { *m = ImportColumnAttrsReque func (m *ImportColumnAttrsRequest) String() string { return proto.CompactTextString(m) } func (*ImportColumnAttrsRequest) ProtoMessage() {} func (*ImportColumnAttrsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_48374b395a722341, []int{21} + return fileDescriptor_413a91106d7bcce8, []int{23} } func (m *ImportColumnAttrsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1482,15 +1593,15 @@ func (m *ImportColumnAttrsRequest) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_ImportColumnAttrsRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ImportColumnAttrsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportColumnAttrsRequest.Merge(dst, src) +func (m *ImportColumnAttrsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportColumnAttrsRequest.Merge(m, src) } func (m *ImportColumnAttrsRequest) XXX_Size() int { return m.Size() @@ -1556,14 +1667,91 @@ func init() { proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") proto.RegisterType((*TranslateKeysRequest)(nil), "internal.TranslateKeysRequest") proto.RegisterType((*TranslateKeysResponse)(nil), "internal.TranslateKeysResponse") + proto.RegisterType((*TranslateIDsRequest)(nil), "internal.TranslateIDsRequest") + proto.RegisterType((*TranslateIDsResponse)(nil), "internal.TranslateIDsResponse") proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") proto.RegisterType((*ImportColumnAttrsRequest)(nil), "internal.ImportColumnAttrsRequest") } + +func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } + +var fileDescriptor_413a91106d7bcce8 = []byte{ + // 1084 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x8e, 0x1b, 0x45, + 0x10, 0xa6, 0x3d, 0xe3, 0xf5, 0xb8, 0xbc, 0xbb, 0x44, 0x1d, 0x27, 0x8c, 0x50, 0xb4, 0xb1, 0x5a, + 0x11, 0x1a, 0x38, 0x6c, 0x94, 0x05, 0xa1, 0x9c, 0xf8, 0x49, 0xec, 0x80, 0x15, 0xc5, 0x0a, 0xed, + 0x95, 0xb9, 0x21, 0xcd, 0x66, 0x3a, 0xce, 0x48, 0xe3, 0x19, 0x33, 0x3f, 0x38, 0xfb, 0x1c, 0x5c, + 0x10, 0x4f, 0xc0, 0x3b, 0xf0, 0x02, 0x9c, 0x10, 0x8f, 0x00, 0xcb, 0x63, 0x70, 0x41, 0x55, 0x3d, + 0xed, 0x1e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xab, 0xbf, 0xae, 0xae, 0xaf, 0xba, 0xfa, 0xeb, 0x86, + 0xfd, 0x65, 0x75, 0x96, 0xc4, 0x2f, 0x8e, 0x97, 0x79, 0x56, 0x66, 0xdc, 0x8b, 0xd3, 0x52, 0xe5, + 0x69, 0x98, 0x88, 0x02, 0x1c, 0x99, 0xad, 0xb8, 0x0f, 0x9d, 0xc7, 0x59, 0x52, 0x2d, 0xd2, 0xc2, + 0x67, 0x03, 0x27, 0x70, 0xa5, 0x51, 0x39, 0x07, 0xf7, 0xa9, 0x3a, 0x2f, 0x7c, 0x67, 0xe0, 0x04, + 0x5d, 0x49, 0x32, 0xbf, 0x07, 0xed, 0x2f, 0xcb, 0x32, 0x2f, 0xfc, 0xd6, 0xc0, 0x09, 0x7a, 0x27, + 0x87, 0xc7, 0x26, 0xdd, 0x31, 0x9a, 0xa5, 0x76, 0x62, 0x4e, 0x99, 0x85, 0x79, 0x9c, 0xce, 0x7d, + 0x77, 0xc0, 0x82, 0x7d, 0x69, 0x54, 0xf1, 0x0c, 0xba, 0xd3, 0x78, 0x9e, 0xaa, 0x08, 0xb7, 0xbe, + 0x0b, 0xce, 0xf3, 0x0c, 0xb7, 0x65, 0x41, 0xef, 0xe4, 0xc0, 0xa6, 0x92, 0xd9, 0x4a, 0xa2, 0x07, + 0x03, 0x26, 0x6a, 0xee, 0xb7, 0x76, 0x06, 0x4c, 0xd4, 0x5c, 0x3c, 0x84, 0x43, 0x99, 0xad, 0xc6, + 0x91, 0x4a, 0xcb, 0xf8, 0x65, 0xac, 0x72, 0x2a, 0x5a, 0x66, 0x2b, 0x83, 0x85, 0xe4, 0x35, 0x90, + 0x96, 0x05, 0x22, 0x3e, 0x03, 0xf7, 0x79, 0x18, 0xe7, 0xfc, 0x10, 0x5a, 0xe3, 0x21, 0x95, 0xe0, + 0xca, 0xd6, 0x78, 0xc8, 0x6f, 0x80, 0xf3, 0x54, 0x9d, 0xfb, 0xce, 0x80, 0x05, 0x5d, 0x89, 0x22, + 0xef, 0x43, 0xfb, 0x71, 0x56, 0xa5, 0x25, 0x95, 0xe1, 0x4a, 0xad, 0x88, 0x11, 0x74, 0x71, 0xfd, + 0x93, 0x58, 0x25, 0x11, 0x17, 0x3a, 0x59, 0x8d, 0xa4, 0xd1, 0x14, 0xb4, 0x4a, 0xbd, 0x51, 0x1f, + 0xda, 0x14, 0x4c, 0x69, 0xba, 0x52, 0x2b, 0xe2, 0x6b, 0x00, 0xf4, 0x16, 0x3a, 0xcf, 0x3d, 0x68, + 0x93, 0x46, 0xd5, 0x5f, 0x4e, 0xa4, 0x9d, 0x57, 0x64, 0x9a, 0x80, 0x47, 0x02, 0x36, 0x76, 0x1d, + 0xc1, 0x1a, 0x11, 0x68, 0xc5, 0x66, 0x0d, 0x0d, 0x10, 0x52, 0xf8, 0x6d, 0xd8, 0x93, 0xd9, 0xca, + 0x62, 0xae, 0x35, 0xf1, 0x1d, 0xc0, 0x57, 0x79, 0x56, 0x2d, 0x09, 0x2e, 0x0f, 0xa0, 0x4d, 0x5a, + 0x5d, 0x19, 0xb7, 0x95, 0x99, 0x4d, 0xa5, 0x0e, 0xd8, 0xdd, 0x2e, 0x6c, 0xeb, 0xb4, 0x5a, 0xd0, + 0x16, 0x8e, 0x44, 0x51, 0x9c, 0x80, 0x37, 0x0b, 0x93, 0xb5, 0x77, 0x16, 0x26, 0x54, 0xad, 0x23, + 0x51, 0xdc, 0xcc, 0xe2, 0x98, 0xa6, 0x7f, 0x0b, 0x07, 0x7a, 0x38, 0x71, 0xcc, 0xa6, 0xaa, 0x7c, + 0x83, 0xd3, 0x7b, 0xa3, 0x81, 0x15, 0xbf, 0x30, 0x70, 0x51, 0x32, 0x09, 0x98, 0x4d, 0xc0, 0xc1, + 0x3d, 0x3d, 0x5f, 0xaa, 0x1a, 0x0e, 0xc9, 0x7c, 0x00, 0xbd, 0x69, 0x89, 0xf3, 0x3c, 0x0b, 0x93, + 0x4a, 0xd5, 0xdb, 0x35, 0x4d, 0xfc, 0x7d, 0xf0, 0xc6, 0x69, 0xa9, 0xdd, 0x2e, 0x41, 0x58, 0xeb, + 0xfc, 0x0e, 0x74, 0x1f, 0x65, 0x59, 0xa2, 0x9d, 0xed, 0x01, 0x0b, 0x3c, 0x69, 0x0d, 0xfc, 0x08, + 0xe0, 0x49, 0x92, 0x85, 0xf5, 0xda, 0xbd, 0x01, 0x0b, 0x98, 0x6c, 0x58, 0xc4, 0x7d, 0xe8, 0x60, + 0xa5, 0xcf, 0xc2, 0xa5, 0xc5, 0xc6, 0xae, 0xc3, 0xf6, 0x0f, 0x83, 0xfd, 0x6f, 0x2a, 0x95, 0x9f, + 0x4b, 0xf5, 0x7d, 0xa5, 0x8a, 0x12, 0x7b, 0x4b, 0xba, 0x99, 0x0e, 0x52, 0x70, 0x0e, 0xa6, 0xaf, + 0xc2, 0x3c, 0xd2, 0x9d, 0x72, 0x65, 0xad, 0x21, 0x56, 0xdb, 0xf3, 0x82, 0xb0, 0x7a, 0xb2, 0x69, + 0xa2, 0x09, 0x52, 0x8b, 0xac, 0x34, 0x60, 0x6a, 0x8d, 0x07, 0xf0, 0xee, 0xe8, 0xf5, 0x8b, 0xa4, + 0x8a, 0x94, 0xcc, 0x56, 0x7a, 0xf5, 0x1e, 0x05, 0x6c, 0x9b, 0xf9, 0x07, 0x70, 0x58, 0x9b, 0x0c, + 0x15, 0x75, 0x28, 0x70, 0xcb, 0xca, 0x1f, 0xc0, 0xfe, 0x68, 0x71, 0xa6, 0xa2, 0x48, 0x45, 0xc3, + 0xb0, 0x0c, 0x7d, 0x8f, 0x70, 0x6f, 0x11, 0xc3, 0x46, 0x88, 0xf8, 0x91, 0xc1, 0x41, 0x8d, 0xbe, + 0x58, 0x66, 0x69, 0xa1, 0xf0, 0x88, 0x47, 0x79, 0x6e, 0x8e, 0x78, 0x94, 0xe7, 0xfc, 0x3e, 0x74, + 0xa4, 0x2a, 0xaa, 0xa4, 0x34, 0x53, 0x72, 0xcb, 0x66, 0x34, 0x6b, 0xab, 0xa4, 0x94, 0x26, 0x8a, + 0x7f, 0x0e, 0x87, 0x1b, 0x73, 0xa8, 0x39, 0xb2, 0x77, 0xf2, 0x9e, 0x5d, 0xb7, 0xe1, 0x97, 0x5b, + 0xe1, 0xe2, 0x57, 0x07, 0x7a, 0x8d, 0xcc, 0xeb, 0x21, 0xc3, 0xfe, 0x1c, 0xd4, 0x43, 0x76, 0x97, + 0xf8, 0xf9, 0x0a, 0x76, 0xc4, 0x5b, 0xbe, 0x0f, 0x6c, 0x52, 0x8f, 0x25, 0x9b, 0x58, 0xee, 0x70, + 0xae, 0xe3, 0x0e, 0x64, 0xfb, 0x57, 0x61, 0x3a, 0x57, 0x11, 0x8d, 0xa5, 0x27, 0x8d, 0xca, 0x8f, + 0xed, 0x7d, 0xa4, 0x73, 0xdc, 0xb8, 0xe4, 0xc6, 0x23, 0xed, 0x9d, 0xd5, 0xbc, 0x31, 0x1e, 0xe2, + 0x59, 0xd1, 0xbc, 0x68, 0x8d, 0x7f, 0x0a, 0x3d, 0xcb, 0x1b, 0x45, 0x7d, 0x44, 0x7d, 0x9b, 0xca, + 0x3a, 0x65, 0x33, 0x90, 0x7f, 0xb1, 0x4d, 0xe5, 0x7e, 0x97, 0xaa, 0xf0, 0x37, 0x90, 0x37, 0xfc, + 0x72, 0x9b, 0xfa, 0x1f, 0x34, 0xde, 0x16, 0x1f, 0x68, 0xf1, 0x4d, 0xbb, 0x78, 0xed, 0x92, 0x8d, + 0x17, 0xe8, 0x93, 0x26, 0xfd, 0xfa, 0x3d, 0x5a, 0xd3, 0xdf, 0xec, 0x9c, 0xf6, 0xc9, 0x46, 0x9c, + 0xf8, 0x8b, 0xc1, 0xc1, 0x78, 0xb1, 0xcc, 0xf2, 0xb2, 0x71, 0xa5, 0xc6, 0x69, 0xa4, 0x5e, 0x9b, + 0x2b, 0x45, 0xca, 0x6e, 0xa2, 0x46, 0x2b, 0x5d, 0x2d, 0xba, 0x4a, 0xae, 0xd4, 0x4a, 0xa3, 0x9d, + 0xee, 0x46, 0x3b, 0xef, 0x40, 0x57, 0xcf, 0x0e, 0xba, 0xda, 0xe4, 0xb2, 0x06, 0xfd, 0xd0, 0xae, + 0xe8, 0x71, 0xeb, 0xd0, 0xe3, 0x66, 0x54, 0xa4, 0x11, 0x1d, 0x46, 0x4e, 0x8f, 0x9c, 0x0d, 0x0b, + 0xfa, 0x4f, 0xe3, 0x85, 0x2a, 0xca, 0x70, 0xb1, 0xc4, 0x7b, 0xe9, 0x04, 0x8e, 0x6c, 0x58, 0xc4, + 0xef, 0x0c, 0xb8, 0xc6, 0x48, 0xb4, 0xf3, 0xff, 0x01, 0xbd, 0x1e, 0xd0, 0x66, 0xd9, 0x9d, 0x4b, + 0x65, 0xdf, 0x86, 0x3d, 0xaa, 0xc7, 0x94, 0x5c, 0x6b, 0xc8, 0x52, 0x96, 0x23, 0x35, 0x5e, 0x26, + 0x9b, 0x26, 0x31, 0x83, 0xfe, 0x69, 0x1e, 0xa6, 0x45, 0x12, 0x96, 0x0a, 0x53, 0xbd, 0x0d, 0xa2, + 0x1d, 0x3f, 0x22, 0xf1, 0x21, 0xdc, 0xda, 0xca, 0x6b, 0x79, 0x06, 0x21, 0x3a, 0x04, 0x11, 0x45, + 0x31, 0x85, 0x9b, 0xeb, 0xd0, 0xf1, 0xf0, 0xad, 0x2a, 0xb8, 0x9c, 0xf4, 0xa3, 0x06, 0x2e, 0x4a, + 0x5a, 0x6f, 0xbf, 0xab, 0xd6, 0x47, 0xe0, 0xd7, 0x73, 0xab, 0xbf, 0x63, 0x75, 0x05, 0xb3, 0x58, + 0xad, 0x30, 0x7e, 0x12, 0x2e, 0x54, 0x5d, 0x04, 0xc9, 0x68, 0x23, 0x9e, 0x6d, 0xd1, 0x27, 0x8e, + 0x64, 0xf1, 0x12, 0xfa, 0xbb, 0x72, 0xd0, 0x8b, 0x9d, 0xa8, 0x50, 0x13, 0xab, 0x27, 0xb5, 0xc2, + 0x1f, 0x42, 0xfb, 0x87, 0x58, 0xad, 0x0c, 0xb1, 0x0a, 0x7b, 0xb7, 0xae, 0x2a, 0x44, 0xea, 0x05, + 0xe2, 0x67, 0x66, 0x8a, 0x6d, 0xbc, 0x35, 0xff, 0xd9, 0x32, 0x3d, 0x70, 0xf5, 0xa7, 0x41, 0x0f, + 0x9c, 0xaf, 0x1f, 0x4c, 0xfb, 0x2f, 0x30, 0x2a, 0x3e, 0xd2, 0x28, 0xce, 0xc2, 0x44, 0xdf, 0xba, + 0xae, 0x5c, 0xeb, 0xd7, 0x8f, 0xe9, 0xa3, 0x1b, 0xbf, 0x5d, 0x1c, 0xb1, 0x3f, 0x2e, 0x8e, 0xd8, + 0x9f, 0x17, 0x47, 0xec, 0xa7, 0xbf, 0x8f, 0xde, 0x39, 0xdb, 0xa3, 0xef, 0xf5, 0xc7, 0xff, 0x06, + 0x00, 0x00, 0xff, 0xff, 0x1a, 0x84, 0x0d, 0x7c, 0x6e, 0x0b, 0x00, 0x00, +} + func (m *Row) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1571,10 +1759,49 @@ func (m *Row) Marshal() (dAtA []byte, err error) { } func (m *Row) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Roaring) > 0 { + i -= len(m.Roaring) + copy(dAtA[i:], m.Roaring) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Roaring))) + i-- + dAtA[i] = 0x22 + } + if len(m.Keys) > 0 { + for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Keys[iNdEx]) + copy(dAtA[i:], m.Keys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Attrs) > 0 { + for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } if len(m.Columns) > 0 { dAtA2 := make([]byte, len(m.Columns)*10) var j1 int @@ -1587,54 +1814,19 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { dAtA2[j1] = uint8(num) j1++ } - dAtA[i] = 0xa - i++ + i -= j1 + copy(dAtA[i:], dAtA2[:j1]) i = encodeVarintPublic(dAtA, i, uint64(j1)) - i += copy(dAtA[i:], dAtA2[:j1]) + i-- + dAtA[i] = 0xa } - if len(m.Attrs) > 0 { - for _, msg := range m.Attrs { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if len(m.Keys) > 0 { - for _, s := range m.Keys { - dAtA[i] = 0x1a - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } - } - if len(m.Roaring) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Roaring))) - i += copy(dAtA[i:], m.Roaring) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *SignedRow) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1642,40 +1834,50 @@ func (m *SignedRow) Marshal() (dAtA []byte, err error) { } func (m *SignedRow) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SignedRow) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Pos != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Pos.Size())) - n3, err := m.Pos.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n3 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Neg != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Neg.Size())) - n4, err := m.Neg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Neg.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) } - i += n4 + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Pos != nil { + { + size, err := m.Pos.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1683,10 +1885,28 @@ func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { } func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RowIdentifiers) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Keys) > 0 { + for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Keys[iNdEx]) + copy(dAtA[i:], m.Keys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } if len(m.Rows) > 0 { dAtA6 := make([]byte, len(m.Rows)*10) var j5 int @@ -1699,36 +1919,19 @@ func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { dAtA6[j5] = uint8(num) j5++ } - dAtA[i] = 0xa - i++ + i -= j5 + copy(dAtA[i:], dAtA6[:j5]) i = encodeVarintPublic(dAtA, i, uint64(j5)) - i += copy(dAtA[i:], dAtA6[:j5]) + i-- + dAtA[i] = 0xa } - if len(m.Keys) > 0 { - for _, s := range m.Keys { - dAtA[i] = 0x12 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *Pair) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1736,36 +1939,43 @@ func (m *Pair) Marshal() (dAtA []byte, err error) { } func (m *Pair) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Pair) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.ID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ID)) - } - if m.Count != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Key) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Key) + copy(dAtA[i:], m.Key) i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i += copy(dAtA[i:], m.Key) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Count != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + i-- + dAtA[i] = 0x10 } - return i, nil + if m.ID != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.ID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func (m *PairField) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1773,36 +1983,45 @@ func (m *PairField) Marshal() (dAtA []byte, err error) { } func (m *PairField) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PairField) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Pair != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Pair.Size())) - n7, err := m.Pair.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n7 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ + i -= len(m.Field) + copy(dAtA[i:], m.Field) i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Pair != nil { + { + size, err := m.Pair.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *PairsField) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1810,38 +2029,47 @@ func (m *PairsField) Marshal() (dAtA []byte, err error) { } func (m *PairsField) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PairsField) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Pairs) > 0 { - for _, msg := range m.Pairs { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ + i -= len(m.Field) + copy(dAtA[i:], m.Field) i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Pairs) > 0 { + for iNdEx := len(m.Pairs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Pairs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } } - return i, nil + return len(dAtA) - i, nil } func (m *FieldRow) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1849,37 +2077,45 @@ func (m *FieldRow) Marshal() (dAtA []byte, err error) { } func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldRow) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Field) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } - if m.RowID != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.RowKey) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.RowKey) + copy(dAtA[i:], m.RowKey) i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) - i += copy(dAtA[i:], m.RowKey) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.RowID != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) + i-- + dAtA[i] = 0x10 } - return i, nil + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *GroupCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1887,42 +2123,50 @@ func (m *GroupCount) Marshal() (dAtA []byte, err error) { } func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GroupCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Group) > 0 { - for _, msg := range m.Group { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.Count != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Sum != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Count != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + i-- + dAtA[i] = 0x10 } - return i, nil + if len(m.Group) > 0 { + for iNdEx := len(m.Group) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Group[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil } func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1930,30 +2174,36 @@ func (m *ValCount) Marshal() (dAtA []byte, err error) { } func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ValCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Val != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Val)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Count != 0 { - dAtA[i] = 0x10 - i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Val != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1961,43 +2211,52 @@ func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { } func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ColumnAttrSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.ID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ID)) - } - if len(m.Attrs) > 0 { - for _, msg := range m.Attrs { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Key) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Key) + copy(dAtA[i:], m.Key) i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i += copy(dAtA[i:], m.Key) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Attrs) > 0 { + for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } } - return i, nil + if m.ID != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.ID)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func (m *Attr) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2005,58 +2264,66 @@ func (m *Attr) Marshal() (dAtA []byte, err error) { } func (m *Attr) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Attr) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Key) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) - i += copy(dAtA[i:], m.Key) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Type != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Type)) - } - if len(m.StringValue) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.StringValue))) - i += copy(dAtA[i:], m.StringValue) - } - if m.IntValue != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.IntValue)) + if m.FloatValue != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i-- + dAtA[i] = 0x31 } if m.BoolValue { - dAtA[i] = 0x28 - i++ + i-- if m.BoolValue { dAtA[i] = 1 } else { dAtA[i] = 0 } - i++ + i-- + dAtA[i] = 0x28 } - if m.FloatValue != 0 { - dAtA[i] = 0x31 - i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i += 8 + if m.IntValue != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IntValue)) + i-- + dAtA[i] = 0x20 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.StringValue) > 0 { + i -= len(m.StringValue) + copy(dAtA[i:], m.StringValue) + i = encodeVarintPublic(dAtA, i, uint64(len(m.StringValue))) + i-- + dAtA[i] = 0x1a } - return i, nil + if m.Type != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Type)) + i-- + dAtA[i] = 0x10 + } + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *AttrMap) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2064,32 +2331,40 @@ func (m *AttrMap) Marshal() (dAtA []byte, err error) { } func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AttrMap) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.Attrs) > 0 { - for _, msg := range m.Attrs { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Attrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Attrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0xa } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *QueryRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2097,15 +2372,72 @@ func (m *QueryRequest) Marshal() (dAtA []byte, err error) { } func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Query) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) - i += copy(dAtA[i:], m.Query) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.EmbeddedData) > 0 { + for iNdEx := len(m.EmbeddedData) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.EmbeddedData[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } + } + if m.ExcludeColumns { + i-- + if m.ExcludeColumns { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x38 + } + if m.ExcludeRowAttrs { + i-- + if m.ExcludeRowAttrs { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } + if m.Remote { + i-- + if m.Remote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x28 + } + if m.ColumnAttrs { + i-- + if m.ColumnAttrs { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 } if len(m.Shards) > 0 { dAtA9 := make([]byte, len(m.Shards)*10) @@ -2119,73 +2451,26 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { dAtA9[j8] = uint8(num) j8++ } - dAtA[i] = 0x12 - i++ + i -= j8 + copy(dAtA[i:], dAtA9[:j8]) i = encodeVarintPublic(dAtA, i, uint64(j8)) - i += copy(dAtA[i:], dAtA9[:j8]) + i-- + dAtA[i] = 0x12 } - if m.ColumnAttrs { - dAtA[i] = 0x18 - i++ - if m.ColumnAttrs { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ + if len(m.Query) > 0 { + i -= len(m.Query) + copy(dAtA[i:], m.Query) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) + i-- + dAtA[i] = 0xa } - if m.Remote { - dAtA[i] = 0x28 - i++ - if m.Remote { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if m.ExcludeRowAttrs { - dAtA[i] = 0x30 - i++ - if m.ExcludeRowAttrs { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if m.ExcludeColumns { - dAtA[i] = 0x38 - i++ - if m.ExcludeColumns { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if len(m.EmbeddedData) > 0 { - for _, msg := range m.EmbeddedData { - dAtA[i] = 0x42 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *QueryResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2193,50 +2478,61 @@ func (m *QueryResponse) Marshal() (dAtA []byte, err error) { } func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueryResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Err) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Err))) - i += copy(dAtA[i:], m.Err) - } - if len(m.Results) > 0 { - for _, msg := range m.Results { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.ColumnAttrSets) > 0 { - for _, msg := range m.ColumnAttrSets { - dAtA[i] = 0x1a - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.ColumnAttrSets) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ColumnAttrSets[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x1a } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Results[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } } - return i, nil + if len(m.Err) > 0 { + i -= len(m.Err) + copy(dAtA[i:], m.Err) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Err))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *QueryResult) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2244,131 +2540,152 @@ func (m *QueryResult) Marshal() (dAtA []byte, err error) { } func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Row != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) - n10, err := m.Row.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n10 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.N != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.N)) - } - if len(m.Pairs) > 0 { - for _, msg := range m.Pairs { - dAtA[i] = 0x1a - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.PairsField != nil { + { + size, err := m.PairsField.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a + } + if m.SignedRow != nil { + { + size, err := m.SignedRow.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } + if m.RowIdentifiers != nil { + { + size, err := m.RowIdentifiers.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x4a + } + if len(m.GroupCounts) > 0 { + for iNdEx := len(m.GroupCounts) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.GroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 } } + if len(m.RowIDs) > 0 { + dAtA14 := make([]byte, len(m.RowIDs)*10) + var j13 int + for _, num := range m.RowIDs { + for num >= 1<<7 { + dAtA14[j13] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j13++ + } + dAtA14[j13] = uint8(num) + j13++ + } + i -= j13 + copy(dAtA[i:], dAtA14[:j13]) + i = encodeVarintPublic(dAtA, i, uint64(j13)) + i-- + dAtA[i] = 0x3a + } + if m.Type != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Type)) + i-- + dAtA[i] = 0x30 + } + if m.ValCount != nil { + { + size, err := m.ValCount.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if m.Changed { - dAtA[i] = 0x20 - i++ + i-- if m.Changed { dAtA[i] = 1 } else { dAtA[i] = 0 } - i++ + i-- + dAtA[i] = 0x20 } - if m.ValCount != nil { - dAtA[i] = 0x2a - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) - n11, err := m.ValCount.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n11 - } - if m.Type != 0 { - dAtA[i] = 0x30 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Type)) - } - if len(m.RowIDs) > 0 { - dAtA13 := make([]byte, len(m.RowIDs)*10) - var j12 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j12++ + if len(m.Pairs) > 0 { + for iNdEx := len(m.Pairs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Pairs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) } - dAtA13[j12] = uint8(num) - j12++ + i-- + dAtA[i] = 0x1a } - dAtA[i] = 0x3a - i++ - i = encodeVarintPublic(dAtA, i, uint64(j12)) - i += copy(dAtA[i:], dAtA13[:j12]) } - if len(m.GroupCounts) > 0 { - for _, msg := range m.GroupCounts { - dAtA[i] = 0x42 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.N != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.N)) + i-- + dAtA[i] = 0x10 + } + if m.Row != nil { + { + size, err := m.Row.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0xa } - if m.RowIdentifiers != nil { - dAtA[i] = 0x4a - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) - n14, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n14 - } - if m.SignedRow != nil { - dAtA[i] = 0x52 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.SignedRow.Size())) - n15, err := m.SignedRow.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n15 - } - if m.PairsField != nil { - dAtA[i] = 0x5a - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.PairsField.Size())) - n16, err := m.PairsField.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n16 - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *ImportRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2376,31 +2693,42 @@ func (m *ImportRequest) Marshal() (dAtA []byte, err error) { } func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if len(m.ColumnKeys) > 0 { + for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ColumnKeys[iNdEx]) + copy(dAtA[i:], m.ColumnKeys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.ColumnKeys[iNdEx]))) + i-- + dAtA[i] = 0x42 + } } - if m.Shard != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + if len(m.RowKeys) > 0 { + for iNdEx := len(m.RowKeys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RowKeys[iNdEx]) + copy(dAtA[i:], m.RowKeys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKeys[iNdEx]))) + i-- + dAtA[i] = 0x3a + } } - if len(m.RowIDs) > 0 { - dAtA18 := make([]byte, len(m.RowIDs)*10) + if len(m.Timestamps) > 0 { + dAtA18 := make([]byte, len(m.Timestamps)*10) var j17 int - for _, num := range m.RowIDs { + for _, num1 := range m.Timestamps { + num := uint64(num1) for num >= 1<<7 { dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2409,10 +2737,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA18[j17] = uint8(num) j17++ } - dAtA[i] = 0x22 - i++ + i -= j17 + copy(dAtA[i:], dAtA18[:j17]) i = encodeVarintPublic(dAtA, i, uint64(j17)) - i += copy(dAtA[i:], dAtA18[:j17]) + i-- + dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { dAtA20 := make([]byte, len(m.ColumnIDs)*10) @@ -2426,16 +2755,16 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA20[j19] = uint8(num) j19++ } - dAtA[i] = 0x2a - i++ + i -= j19 + copy(dAtA[i:], dAtA20[:j19]) i = encodeVarintPublic(dAtA, i, uint64(j19)) - i += copy(dAtA[i:], dAtA20[:j19]) + i-- + dAtA[i] = 0x2a } - if len(m.Timestamps) > 0 { - dAtA22 := make([]byte, len(m.Timestamps)*10) + if len(m.RowIDs) > 0 { + dAtA22 := make([]byte, len(m.RowIDs)*10) var j21 int - for _, num1 := range m.Timestamps { - num := uint64(num1) + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2444,51 +2773,38 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA22[j21] = uint8(num) j21++ } - dAtA[i] = 0x32 - i++ + i -= j21 + copy(dAtA[i:], dAtA22[:j21]) i = encodeVarintPublic(dAtA, i, uint64(j21)) - i += copy(dAtA[i:], dAtA22[:j21]) + i-- + dAtA[i] = 0x22 } - if len(m.RowKeys) > 0 { - for _, s := range m.RowKeys { - dAtA[i] = 0x3a - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } + if m.Shard != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x18 } - if len(m.ColumnKeys) > 0 { - for _, s := range m.ColumnKeys { - dAtA[i] = 0x42 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *ImportValueRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2496,97 +2812,101 @@ func (m *ImportValueRequest) Marshal() (dAtA []byte, err error) { } func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } - if m.Shard != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) - } - if len(m.ColumnIDs) > 0 { - dAtA24 := make([]byte, len(m.ColumnIDs)*10) - var j23 int - for _, num := range m.ColumnIDs { - for num >= 1<<7 { - dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j23++ - } - dAtA24[j23] = uint8(num) - j23++ + if len(m.FloatValues) > 0 { + for iNdEx := len(m.FloatValues) - 1; iNdEx >= 0; iNdEx-- { + f23 := math.Float64bits(float64(m.FloatValues[iNdEx])) + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f23)) + } + i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) + i-- + dAtA[i] = 0x42 + } + if len(m.ColumnKeys) > 0 { + for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ColumnKeys[iNdEx]) + copy(dAtA[i:], m.ColumnKeys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.ColumnKeys[iNdEx]))) + i-- + dAtA[i] = 0x3a } - dAtA[i] = 0x2a - i++ - i = encodeVarintPublic(dAtA, i, uint64(j23)) - i += copy(dAtA[i:], dAtA24[:j23]) } if len(m.Values) > 0 { - dAtA26 := make([]byte, len(m.Values)*10) - var j25 int + dAtA25 := make([]byte, len(m.Values)*10) + var j24 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA26[j25] = uint8(uint64(num)&0x7f | 0x80) + dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j25++ + j24++ } - dAtA26[j25] = uint8(num) - j25++ + dAtA25[j24] = uint8(num) + j24++ } + i -= j24 + copy(dAtA[i:], dAtA25[:j24]) + i = encodeVarintPublic(dAtA, i, uint64(j24)) + i-- dAtA[i] = 0x32 - i++ - i = encodeVarintPublic(dAtA, i, uint64(j25)) - i += copy(dAtA[i:], dAtA26[:j25]) } - if len(m.ColumnKeys) > 0 { - for _, s := range m.ColumnKeys { - dAtA[i] = 0x3a - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + if len(m.ColumnIDs) > 0 { + dAtA27 := make([]byte, len(m.ColumnIDs)*10) + var j26 int + for _, num := range m.ColumnIDs { + for num >= 1<<7 { + dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j26++ } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + dAtA27[j26] = uint8(num) + j26++ } + i -= j26 + copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintPublic(dAtA, i, uint64(j26)) + i-- + dAtA[i] = 0x2a } - if len(m.FloatValues) > 0 { - dAtA[i] = 0x42 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) - for _, num := range m.FloatValues { - f27 := math.Float64bits(float64(num)) - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f27)) - i += 8 - } + if m.Shard != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2594,47 +2914,49 @@ func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { } func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TranslateKeysRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Field) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Keys) > 0 { - for _, s := range m.Keys { + for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Keys[iNdEx]) + copy(dAtA[i:], m.Keys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) + i-- dAtA[i] = 0x1a - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -2642,10 +2964,19 @@ func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { } func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.IDs) > 0 { dAtA29 := make([]byte, len(m.IDs)*10) var j28 int @@ -2658,144 +2989,43 @@ func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { dAtA29[j28] = uint8(num) j28++ } - dAtA[i] = 0x1a - i++ + i -= j28 + copy(dAtA[i:], dAtA29[:j28]) i = encodeVarintPublic(dAtA, i, uint64(j28)) - i += copy(dAtA[i:], dAtA29[:j28]) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - -func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } - if len(m.Data) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) - i += copy(dAtA[i:], m.Data) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - -func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Clear { - dAtA[i] = 0x8 - i++ - if m.Clear { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if len(m.Views) > 0 { - for _, msg := range m.Views { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - -func (m *ImportColumnAttrsRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ImportColumnAttrsRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if m.Shard != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) - } - if len(m.AttrKey) > 0 { + i-- dAtA[i] = 0x1a - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.AttrKey))) - i += copy(dAtA[i:], m.AttrKey) } - if len(m.AttrVals) > 0 { - for _, s := range m.AttrVals { - dAtA[i] = 0x22 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } + return len(dAtA) - i, nil +} + +func (m *TranslateIDsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - if len(m.ColumnIDs) > 0 { - dAtA31 := make([]byte, len(m.ColumnIDs)*10) + return dAtA[:n], nil +} + +func (m *TranslateIDsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TranslateIDsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.IDs) > 0 { + dAtA31 := make([]byte, len(m.IDs)*10) var j30 int - for _, num := range m.ColumnIDs { + for _, num := range m.IDs { for num >= 1<<7 { dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2804,25 +3034,240 @@ func (m *ImportColumnAttrsRequest) MarshalTo(dAtA []byte) (int, error) { dAtA31[j30] = uint8(num) j30++ } - dAtA[i] = 0x2a - i++ + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) i = encodeVarintPublic(dAtA, i, uint64(j30)) - i += copy(dAtA[i:], dAtA31[:j30]) + i-- + dAtA[i] = 0x1a } + if len(m.Field) > 0 { + i -= len(m.Field) + copy(dAtA[i:], m.Field) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i-- + dAtA[i] = 0x12 + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *TranslateIDsResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateIDsResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TranslateIDsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + if len(m.Keys) > 0 { + for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Keys[iNdEx]) + copy(dAtA[i:], m.Keys[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Keys[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + return len(dAtA) - i, nil +} + +func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportRoaringRequestView) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0x12 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Views) > 0 { + for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Views[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if m.Clear { + i-- + if m.Clear { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ImportColumnAttrsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportColumnAttrsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.ColumnIDs) > 0 { + dAtA33 := make([]byte, len(m.ColumnIDs)*10) + var j32 int + for _, num := range m.ColumnIDs { + for num >= 1<<7 { + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j32++ + } + dAtA33[j32] = uint8(num) + j32++ + } + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPublic(dAtA, i, uint64(j32)) + i-- + dAtA[i] = 0x2a + } + if len(m.AttrVals) > 0 { + for iNdEx := len(m.AttrVals) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.AttrVals[iNdEx]) + copy(dAtA[i:], m.AttrVals[iNdEx]) + i = encodeVarintPublic(dAtA, i, uint64(len(m.AttrVals[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if len(m.AttrKey) > 0 { + i -= len(m.AttrKey) + copy(dAtA[i:], m.AttrKey) + i = encodeVarintPublic(dAtA, i, uint64(len(m.AttrKey))) + i-- + dAtA[i] = 0x1a + } + if m.Shard != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x10 + } + if len(m.Index) > 0 { + i -= len(m.Index) + copy(dAtA[i:], m.Index) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { + offset -= sovPublic(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func (m *Row) Size() (n int) { if m == nil { @@ -3384,6 +3829,51 @@ func (m *TranslateKeysResponse) Size() (n int) { return n } +func (m *TranslateIDsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if len(m.IDs) > 0 { + l = 0 + for _, e := range m.IDs { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *TranslateIDsResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *ImportRoaringRequestView) Size() (n int) { if m == nil { return 0 @@ -3462,14 +3952,7 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { } func sovPublic(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozPublic(x uint64) (n int) { return sovPublic(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -3489,7 +3972,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3515,7 +3998,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3532,7 +4015,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3541,12 +4024,15 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -3566,7 +4052,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3590,7 +4076,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3599,6 +4085,9 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3621,7 +4110,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3631,6 +4120,9 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3650,7 +4142,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3659,6 +4151,9 @@ func (m *Row) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3676,6 +4171,9 @@ func (m *Row) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3704,7 +4202,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3732,7 +4230,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3741,6 +4239,9 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3765,7 +4266,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3774,6 +4275,9 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3793,6 +4297,9 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3821,7 +4328,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3847,7 +4354,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3864,7 +4371,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -3873,12 +4380,15 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -3898,7 +4408,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3922,7 +4432,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -3932,6 +4442,9 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -3946,6 +4459,9 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -3974,7 +4490,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4002,7 +4518,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ID |= (uint64(b) & 0x7F) << shift + m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4021,7 +4537,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= (uint64(b) & 0x7F) << shift + m.Count |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4040,7 +4556,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4050,6 +4566,9 @@ func (m *Pair) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4064,6 +4583,9 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4092,7 +4614,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4120,7 +4642,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4129,6 +4651,9 @@ func (m *PairField) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4153,7 +4678,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4163,6 +4688,9 @@ func (m *PairField) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4177,6 +4705,9 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4205,7 +4736,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4233,7 +4764,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4242,6 +4773,9 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4264,7 +4798,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4274,6 +4808,9 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4288,6 +4825,9 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4316,7 +4856,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4344,7 +4884,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4354,6 +4894,9 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4373,7 +4916,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RowID |= (uint64(b) & 0x7F) << shift + m.RowID |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4392,7 +4935,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4402,6 +4945,9 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4416,6 +4962,9 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4444,7 +4993,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4472,7 +5021,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4481,6 +5030,9 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4503,7 +5055,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= (uint64(b) & 0x7F) << shift + m.Count |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4522,7 +5074,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= (int64(b) & 0x7F) << shift + m.Sum |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4536,6 +5088,9 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4564,7 +5119,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4592,7 +5147,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Val |= (int64(b) & 0x7F) << shift + m.Val |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4611,7 +5166,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Count |= (int64(b) & 0x7F) << shift + m.Count |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4625,6 +5180,9 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4653,7 +5211,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4681,7 +5239,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ID |= (uint64(b) & 0x7F) << shift + m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4700,7 +5258,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4709,6 +5267,9 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4731,7 +5292,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4741,6 +5302,9 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4755,6 +5319,9 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4783,7 +5350,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4811,7 +5378,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4821,6 +5388,9 @@ func (m *Attr) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4840,7 +5410,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Type |= (uint64(b) & 0x7F) << shift + m.Type |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4859,7 +5429,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4869,6 +5439,9 @@ func (m *Attr) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -4888,7 +5461,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.IntValue |= (int64(b) & 0x7F) << shift + m.IntValue |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -4907,7 +5480,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4933,6 +5506,9 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -4961,7 +5537,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -4989,7 +5565,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -4998,6 +5574,9 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5015,6 +5594,9 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5043,7 +5625,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5071,7 +5653,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5081,6 +5663,9 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5098,7 +5683,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5115,7 +5700,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5124,12 +5709,15 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -5149,7 +5737,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5173,7 +5761,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5193,7 +5781,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5213,7 +5801,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5233,7 +5821,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5253,7 +5841,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5262,6 +5850,9 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5279,6 +5870,9 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5307,7 +5901,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5335,7 +5929,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5345,6 +5939,9 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5364,7 +5961,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5373,6 +5970,9 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5395,7 +5995,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5404,6 +6004,9 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5421,6 +6024,9 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5449,7 +6055,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5477,7 +6083,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5486,6 +6092,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5510,7 +6119,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.N |= (uint64(b) & 0x7F) << shift + m.N |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5529,7 +6138,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5538,6 +6147,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5560,7 +6172,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5580,7 +6192,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5589,6 +6201,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5613,7 +6228,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Type |= (uint32(b) & 0x7F) << shift + m.Type |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -5630,7 +6245,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5647,7 +6262,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5656,12 +6271,15 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -5681,7 +6299,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5705,7 +6323,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5714,6 +6332,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5736,7 +6357,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5745,6 +6366,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5769,7 +6393,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5778,6 +6402,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5802,7 +6429,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5811,6 +6438,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5830,6 +6460,9 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -5858,7 +6491,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5886,7 +6519,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5896,6 +6529,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5915,7 +6551,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5925,6 +6561,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -5944,7 +6583,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (uint64(b) & 0x7F) << shift + m.Shard |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5961,7 +6600,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5978,7 +6617,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -5987,12 +6626,15 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6012,7 +6654,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6034,7 +6676,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6051,7 +6693,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6060,12 +6702,15 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6085,7 +6730,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6107,7 +6752,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -6124,7 +6769,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6133,12 +6778,15 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6158,7 +6806,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -6182,7 +6830,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6192,6 +6840,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6211,7 +6862,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6221,6 +6872,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6235,6 +6889,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6263,7 +6920,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6291,7 +6948,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6301,6 +6958,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6320,7 +6980,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6330,6 +6990,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6349,7 +7012,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (uint64(b) & 0x7F) << shift + m.Shard |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6366,7 +7029,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6383,7 +7046,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6392,12 +7055,15 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6417,7 +7083,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6439,7 +7105,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -6456,7 +7122,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6465,12 +7131,15 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6490,7 +7159,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -6514,7 +7183,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6524,6 +7193,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6550,7 +7222,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6559,6 +7231,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6589,6 +7264,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6617,7 +7295,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6645,7 +7323,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6655,6 +7333,9 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6674,7 +7355,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6684,6 +7365,9 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6703,7 +7387,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6713,6 +7397,9 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6727,6 +7414,9 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6755,7 +7445,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6781,7 +7471,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6798,7 +7488,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6807,12 +7497,15 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -6832,7 +7525,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6851,6 +7544,289 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateIDsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateIDsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.IDs) == 0 { + m.IDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateIDsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateIDsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6879,7 +7855,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6907,7 +7883,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -6917,6 +7893,9 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6936,7 +7915,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -6945,6 +7924,9 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -6962,6 +7944,9 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -6990,7 +7975,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7018,7 +8003,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7038,7 +8023,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7047,6 +8032,9 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7064,6 +8052,9 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7092,7 +8083,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7120,7 +8111,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7130,6 +8121,9 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7149,7 +8143,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Shard |= (int64(b) & 0x7F) << shift + m.Shard |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7168,7 +8162,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7178,6 +8172,9 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7197,7 +8194,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7207,6 +8204,9 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -7224,7 +8224,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7241,7 +8241,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - packedLen |= (int(b) & 0x7F) << shift + packedLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -7250,12 +8250,15 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { return ErrInvalidLengthPublic } postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int var count int - for _, integer := range dAtA { + for _, integer := range dAtA[iNdEx:postIndex] { if integer < 128 { count++ } @@ -7275,7 +8278,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -7294,6 +8297,9 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthPublic } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPublic + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -7310,6 +8316,7 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -7341,10 +8348,8 @@ func skipPublic(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -7361,126 +8366,34 @@ func skipPublic(dAtA []byte) (n int, err error) { break } } - iNdEx += length if length < 0 { return 0, ErrInvalidLengthPublic } - return iNdEx, nil + iNdEx += length case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPublic - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipPublic(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupPublic + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthPublic + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthPublic = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthPublic = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupPublic = fmt.Errorf("proto: unexpected end of group") ) - -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_48374b395a722341) } - -var fileDescriptor_public_48374b395a722341 = []byte{ - // 1065 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdb, 0x6e, 0x1c, 0x45, - 0x13, 0xfe, 0x7b, 0x67, 0xd6, 0xbb, 0x5b, 0x6b, 0xfb, 0x8f, 0x9a, 0x4d, 0x18, 0xa1, 0xc8, 0xac, - 0x5a, 0x11, 0x1a, 0x6e, 0x1c, 0xc5, 0x20, 0x94, 0x2b, 0x0e, 0x8e, 0x1d, 0x58, 0x45, 0x59, 0x85, - 0xb2, 0xb5, 0xdc, 0x21, 0x8d, 0xb3, 0x1d, 0x67, 0xa4, 0xd9, 0x99, 0x65, 0x0e, 0x4c, 0xfc, 0x1c, - 0xdc, 0x20, 0x9e, 0x80, 0x77, 0xe0, 0x05, 0xb8, 0x42, 0x3c, 0x02, 0x98, 0xc7, 0xe0, 0x06, 0x55, - 0xf5, 0xf4, 0xf6, 0xec, 0xfa, 0x00, 0x42, 0xdc, 0xf5, 0x57, 0xa7, 0xa9, 0xaa, 0xae, 0xfa, 0x7a, - 0x60, 0x7b, 0x59, 0x9d, 0x25, 0xf1, 0xcb, 0xfd, 0x65, 0x9e, 0x95, 0x99, 0xec, 0xc7, 0x69, 0xa9, - 0xf3, 0x34, 0x4a, 0x54, 0x01, 0x1e, 0x66, 0xb5, 0x0c, 0xa0, 0xf7, 0x24, 0x4b, 0xaa, 0x45, 0x5a, - 0x04, 0x62, 0xec, 0x85, 0x3e, 0x5a, 0x28, 0x1f, 0x40, 0xf7, 0xb3, 0xb2, 0xcc, 0x8b, 0xa0, 0x33, - 0xf6, 0xc2, 0xe1, 0xc1, 0xee, 0xbe, 0x75, 0xdd, 0x27, 0x31, 0x1a, 0xa5, 0x94, 0xe0, 0x3f, 0xd3, - 0x17, 0x45, 0xe0, 0x8d, 0xbd, 0x70, 0x80, 0x7c, 0xa6, 0x98, 0x98, 0x45, 0x79, 0x9c, 0x9e, 0x07, - 0xfe, 0x58, 0x84, 0xdb, 0x68, 0xa1, 0x7a, 0x0e, 0x83, 0x93, 0xf8, 0x3c, 0xd5, 0x73, 0xfa, 0xf4, - 0xbb, 0xe0, 0xbd, 0xc8, 0xe8, 0xb3, 0x22, 0x1c, 0x1e, 0xec, 0xb8, 0xf0, 0x98, 0xd5, 0x48, 0x1a, - 0x32, 0x98, 0xea, 0xf3, 0xa0, 0x73, 0xad, 0xc1, 0x54, 0x9f, 0xab, 0xc7, 0xb0, 0x8b, 0x59, 0x3d, - 0x99, 0xeb, 0xb4, 0x8c, 0x5f, 0xc5, 0xda, 0xa4, 0x83, 0x59, 0x6d, 0x6b, 0xe1, 0xf3, 0x2a, 0xc5, - 0x8e, 0x4b, 0x51, 0x7d, 0x0c, 0xfe, 0x8b, 0x28, 0xce, 0xe5, 0x2e, 0x74, 0x26, 0x47, 0x9c, 0x82, - 0x8f, 0x9d, 0xc9, 0x91, 0x1c, 0x41, 0xf7, 0x49, 0x56, 0xa5, 0x25, 0x7f, 0xd4, 0x47, 0x03, 0xe4, - 0x1d, 0xf0, 0x9e, 0xe9, 0x8b, 0xc0, 0x1b, 0x8b, 0x70, 0x80, 0x74, 0x54, 0xc7, 0x30, 0x20, 0xff, - 0xa7, 0xb1, 0x4e, 0xe6, 0x52, 0x99, 0x60, 0x4d, 0x25, 0xad, 0x46, 0x91, 0x14, 0xcd, 0x87, 0x46, - 0xd0, 0x65, 0x63, 0x0e, 0x3c, 0x40, 0x03, 0xd4, 0x17, 0x00, 0xa4, 0x2d, 0x4c, 0x9c, 0x07, 0xd0, - 0x65, 0xc4, 0xd9, 0x5f, 0x0d, 0x64, 0x94, 0x37, 0x44, 0x9a, 0x42, 0x9f, 0x0f, 0xd4, 0xd8, 0x95, - 0x85, 0x68, 0x59, 0x90, 0x94, 0x9a, 0x75, 0x64, 0x4b, 0x63, 0x20, 0xef, 0xc1, 0x16, 0x66, 0xb5, - 0xab, 0xae, 0x41, 0xea, 0x6b, 0x80, 0xcf, 0xf3, 0xac, 0x5a, 0x9a, 0x06, 0x84, 0xd0, 0x65, 0xd4, - 0x64, 0x26, 0x5d, 0x66, 0xf6, 0xa3, 0x68, 0x0c, 0x6e, 0x6e, 0xe0, 0x49, 0xb5, 0xe0, 0x4f, 0x78, - 0x48, 0x47, 0x75, 0x00, 0xfd, 0x59, 0x94, 0xac, 0xb4, 0xb3, 0x28, 0xe1, 0x6c, 0x3d, 0xa4, 0xe3, - 0x7a, 0x14, 0xaf, 0x89, 0xa2, 0xbe, 0x82, 0x1d, 0x33, 0x9c, 0x34, 0x7a, 0x27, 0xba, 0xbc, 0x72, - 0x7b, 0xff, 0x6c, 0x64, 0xaf, 0xde, 0xe6, 0x8f, 0x02, 0x7c, 0xd2, 0x59, 0x95, 0x58, 0xa9, 0x68, - 0x78, 0x4e, 0x2f, 0x96, 0xba, 0x29, 0x87, 0xcf, 0x72, 0x0c, 0xc3, 0x93, 0x92, 0xe6, 0x79, 0x16, - 0x25, 0x95, 0x6e, 0x02, 0xb5, 0x45, 0xf2, 0x1d, 0xe8, 0x4f, 0xd2, 0xd2, 0xa8, 0x7d, 0x2e, 0x61, - 0x85, 0xe5, 0x7d, 0x18, 0x1c, 0x66, 0x59, 0x62, 0x94, 0xdd, 0xb1, 0x08, 0xfb, 0xe8, 0x04, 0x72, - 0x0f, 0xe0, 0x69, 0x92, 0x45, 0x8d, 0xef, 0xd6, 0x58, 0x84, 0x02, 0x5b, 0x12, 0xf5, 0x10, 0x7a, - 0x94, 0xe9, 0xf3, 0x68, 0xe9, 0xaa, 0x15, 0xb7, 0x54, 0xab, 0xfe, 0x14, 0xb0, 0xfd, 0x65, 0xa5, - 0xf3, 0x0b, 0xd4, 0xdf, 0x54, 0xba, 0x28, 0xa9, 0xb7, 0x8c, 0xed, 0x74, 0x30, 0xa0, 0x39, 0x38, - 0x79, 0x1d, 0xe5, 0x73, 0xd3, 0x3b, 0x1f, 0x1b, 0x44, 0xb5, 0xba, 0x9e, 0x17, 0x5c, 0x6b, 0x1f, - 0xdb, 0x22, 0x9e, 0x20, 0xbd, 0xc8, 0x4a, 0x5b, 0x4c, 0x83, 0x64, 0x08, 0xff, 0x3f, 0x7e, 0xf3, - 0x32, 0xa9, 0xe6, 0x1a, 0xb3, 0xda, 0x78, 0x6f, 0xb1, 0xc1, 0xa6, 0x58, 0xbe, 0x07, 0xbb, 0x8d, - 0xc8, 0x52, 0x51, 0x8f, 0x0d, 0x37, 0xa4, 0xf2, 0x11, 0x6c, 0x1f, 0x2f, 0xce, 0xf4, 0x7c, 0xae, - 0xe7, 0x47, 0x51, 0x19, 0x05, 0x7d, 0xae, 0x7b, 0x83, 0x18, 0xd6, 0x4c, 0xd4, 0x77, 0x02, 0x76, - 0x9a, 0xea, 0x8b, 0x65, 0x96, 0x16, 0x9a, 0xae, 0xf8, 0x38, 0xcf, 0xed, 0x15, 0x1f, 0xe7, 0xb9, - 0x7c, 0x08, 0x3d, 0xd4, 0x45, 0x95, 0x94, 0x76, 0x6e, 0xee, 0xba, 0x88, 0xd6, 0xb7, 0x4a, 0x4a, - 0xb4, 0x56, 0xf2, 0x13, 0xd8, 0x5d, 0x9b, 0x43, 0xc3, 0x7e, 0xc3, 0x83, 0xb7, 0x9d, 0xdf, 0x9a, - 0x1e, 0x37, 0xcc, 0xd5, 0x4f, 0x1e, 0x0c, 0x5b, 0x91, 0x89, 0xe8, 0x30, 0xab, 0x6f, 0x60, 0x42, - 0xda, 0xe8, 0x6d, 0x10, 0xd3, 0x66, 0x04, 0xc5, 0xd4, 0xf1, 0x84, 0x77, 0x1b, 0x4f, 0x10, 0xb3, - 0xbf, 0x8e, 0xd2, 0x73, 0x3d, 0xe7, 0x11, 0xec, 0xa3, 0x85, 0x72, 0xdf, 0xed, 0x1e, 0xdf, 0xd9, - 0xda, 0x42, 0x5b, 0x0d, 0xba, 0xfd, 0xb4, 0x3b, 0x40, 0xd7, 0xb7, 0xd3, 0xec, 0x80, 0xe1, 0x8d, - 0xc9, 0x11, 0xdd, 0x15, 0xcf, 0x8b, 0x41, 0xf2, 0x23, 0x18, 0x3a, 0xde, 0x28, 0x9a, 0x2b, 0x1a, - 0xb9, 0xf0, 0x4e, 0x89, 0x6d, 0x43, 0xf9, 0xe9, 0x26, 0x95, 0x07, 0x03, 0xce, 0x2c, 0x58, 0xeb, - 0x46, 0x4b, 0x8f, 0x9b, 0xd4, 0xff, 0xa8, 0xf5, 0xb6, 0x04, 0xc0, 0xce, 0x6f, 0x39, 0xe7, 0x95, - 0x0a, 0x5b, 0x2f, 0xd0, 0x87, 0x6d, 0xfa, 0x0d, 0x86, 0xec, 0x33, 0x5a, 0xef, 0xa6, 0xd1, 0x61, - 0xcb, 0x4e, 0xfd, 0x2e, 0x60, 0x67, 0xb2, 0x58, 0x66, 0x79, 0xd9, 0x5a, 0xa9, 0x49, 0x3a, 0xd7, - 0x6f, 0xec, 0x4a, 0x31, 0xb8, 0x9e, 0xa8, 0x49, 0xca, 0xab, 0xc5, 0xab, 0xe4, 0xa3, 0x01, 0xad, - 0x76, 0xfa, 0x6b, 0xed, 0xbc, 0x0f, 0x03, 0x33, 0x3b, 0xa4, 0xea, 0xb2, 0xca, 0x09, 0x88, 0x2c, - 0x4e, 0xe3, 0x85, 0x2e, 0xca, 0x68, 0xb1, 0xa4, 0xed, 0xf2, 0x42, 0x0f, 0x5b, 0x12, 0xf3, 0x10, - 0xd7, 0xfc, 0xf8, 0xf5, 0xf8, 0xf1, 0xb3, 0x90, 0x3c, 0x4d, 0x18, 0x56, 0xf6, 0x59, 0xd9, 0x92, - 0xa8, 0x5f, 0x04, 0x48, 0x53, 0x23, 0xd3, 0xce, 0x7f, 0x57, 0xe8, 0xed, 0x05, 0xdd, 0x83, 0x2d, - 0xfe, 0x9e, 0x2d, 0xa6, 0x41, 0x1b, 0xe9, 0xf6, 0x36, 0xd3, 0x25, 0x96, 0x72, 0x1c, 0x69, 0xea, - 0x11, 0xd8, 0x16, 0xa9, 0x19, 0x8c, 0x4e, 0xf3, 0x28, 0x2d, 0x92, 0xa8, 0xd4, 0xe4, 0xf2, 0x6f, - 0x2a, 0xba, 0xe6, 0x5f, 0x47, 0xbd, 0x0f, 0x77, 0x37, 0xe2, 0x3a, 0x9e, 0xa1, 0x12, 0x3d, 0x2e, - 0x91, 0x8e, 0xea, 0x10, 0x82, 0x66, 0x6c, 0xcc, 0xdf, 0x50, 0x93, 0xc2, 0x2c, 0xd6, 0x35, 0x85, - 0x9e, 0x46, 0x0b, 0xdd, 0x64, 0xc1, 0x67, 0x92, 0x31, 0xcd, 0x75, 0xf8, 0x1f, 0x8a, 0xcf, 0xea, - 0x15, 0x8c, 0xae, 0x8b, 0xc1, 0x0f, 0x66, 0xa2, 0x23, 0xc3, 0x6b, 0x7d, 0x34, 0x40, 0x3e, 0x86, - 0xee, 0xb7, 0xb1, 0xae, 0x2d, 0xaf, 0x29, 0x37, 0xda, 0x37, 0x25, 0x82, 0xc6, 0x41, 0xfd, 0x20, - 0x6c, 0xb2, 0x2d, 0xaa, 0xff, 0xdb, 0x9e, 0x99, 0xfb, 0x6e, 0xde, 0x6c, 0x73, 0xdf, 0x81, 0x79, - 0xaf, 0xdc, 0x83, 0x6b, 0x21, 0xbd, 0x91, 0x74, 0x9c, 0x45, 0x89, 0x19, 0xfa, 0x01, 0xae, 0xf0, - 0xed, 0x53, 0x72, 0x78, 0xe7, 0xe7, 0xcb, 0x3d, 0xf1, 0xeb, 0xe5, 0x9e, 0xf8, 0xed, 0x72, 0x4f, - 0x7c, 0xff, 0xc7, 0xde, 0xff, 0xce, 0xb6, 0xf8, 0xef, 0xf6, 0x83, 0xbf, 0x02, 0x00, 0x00, 0xff, - 0xff, 0xdd, 0x9e, 0xfe, 0xf7, 0xed, 0x0a, 0x00, 0x00, -} diff --git a/internal/public.proto b/internal/public.proto index a58c75e96..fcedd24cf 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -132,6 +132,16 @@ message TranslateKeysResponse { repeated uint64 IDs = 3; } +message TranslateIDsRequest { + string Index = 1; + string Field = 2; + repeated uint64 IDs = 3; +} + +message TranslateIDsResponse { + repeated string Keys = 3; +} + message ImportRoaringRequestView { string Name = 1; bytes Data = 2; diff --git a/mock/translator.go b/mock/translator.go index 4291ed43d..9de9fb444 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -25,6 +25,7 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil) type TranslateStore struct { CloseFunc func() error MaxIDFunc func() (uint64, error) + PartitionIDFunc func() int ReadOnlyFunc func() bool SetReadOnlyFunc func(v bool) TranslateKeyFunc func(key string) (uint64, error) @@ -43,6 +44,10 @@ func (s *TranslateStore) MaxID() (uint64, error) { return s.MaxIDFunc() } +func (s *TranslateStore) PartitionID() int { + return s.PartitionIDFunc() +} + func (s *TranslateStore) ReadOnly() bool { return s.ReadOnlyFunc() } diff --git a/pql/ast.go b/pql/ast.go index 738601100..d60b1062c 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -742,6 +742,34 @@ func (c *Call) HasConditionArg() bool { return false } +// TranslateInfo returns the relevant translation fields. +func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) { + switch c.Name { + case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow": + // Positional args in new PQL syntax require special handling here. + fieldName, _ = c.FieldArg() + return "_" + columnLabel, fieldName, fieldName + case "SetRowAttrs": + // Positional args in new PQL syntax require special handling here. + return "", "_" + rowLabel, c.ArgString("_field") + case "Rows": + return "column", "previous", c.ArgString("_field") + case "GroupBy": + return "", "", "" + default: + return "col", "row", c.ArgString("_field") + } +} + +func (c *Call) ArgString(key string) string { + value, ok := c.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} + // Condition represents an operation & value. // When used in an argument map it represents a binary expression. type Condition struct { diff --git a/server.go b/server.go index 96a4d5cde..d89000bb9 100644 --- a/server.go +++ b/server.go @@ -284,7 +284,7 @@ func OptServerClusterHasher(h Hasher) ServerOption { // used to specify the translation data store type. func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption { return func(s *Server) error { - s.holder.OpenTranslateStore = fn + s.cluster.OpenTranslateStore = fn return nil } } @@ -293,7 +293,7 @@ func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption { // used to specify the remote translation data reader. func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { return func(s *Server) error { - s.holder.OpenTranslateReader = fn + s.cluster.OpenTranslateReader = fn return nil } } diff --git a/server/server.go b/server/server.go index e317f9548..e9edeb293 100644 --- a/server/server.go +++ b/server/server.go @@ -137,7 +137,6 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption // Start starts the pilosa server - it returns once the server is running. func (m *Command) Start() (err error) { - // Seed random number generator rand.Seed(time.Now().UTC().UnixNano()) diff --git a/translate.go b/translate.go index 3a2a6891f..cb799698a 100644 --- a/translate.go +++ b/translate.go @@ -28,6 +28,7 @@ var ( ErrTranslateStoreReaderClosed = errors.New("translate store reader closed") ErrReplicationNotSupported = errors.New("replication not supported") ErrTranslateStoreReadOnly = errors.New("translate store could not find or create key, translate store read only") + ErrTranslateStoreNotFound = errors.New("translate store not found") ErrCannotOpenV1TranslateFile = errors.New("cannot open v1 translate .keys file") ) @@ -38,11 +39,18 @@ type TranslateStore interface { // Returns the maximum ID set on the store. MaxID() (uint64, error) + // Retrieves the partition ID associated with the store. + // Only applies to index stores. + PartitionID() int + // Sets & retrieves whether the store is read-only. ReadOnly() bool SetReadOnly(v bool) // Converts a string key to its autoincrementing integer ID value. + // + // Translated id must be associated with a shard in the store's partition + // unless partition is set to -1. TranslateKey(key string) (uint64, error) TranslateKeys(key []string) ([]uint64, error) @@ -58,7 +66,7 @@ type TranslateStore interface { } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. -type OpenTranslateStoreFunc func(path, index, field string) (TranslateStore, error) +type OpenTranslateStoreFunc func(path, index, field string, partitionID int) (TranslateStore, error) // TranslateEntryReader represents a stream of translation entries. type TranslateEntryReader interface { @@ -154,22 +162,22 @@ type readEntryResponse struct { } // TranslateOffsetMap maintains a set of offsets for both indexes & fields. -type TranslateOffsetMap map[string]map[string]uint64 +type TranslateOffsetMap map[string]*IndexTranslateOffsetMap // IndexOffset returns the offset for the given index. -func (m TranslateOffsetMap) IndexOffset(name string) uint64 { +func (m TranslateOffsetMap) IndexPartitionOffset(name string, partitionID int) uint64 { if m[name] == nil { return 0 } - return m[name][""] + return m[name].Partitions[partitionID] } // SetIndexOffset sets the offset for the given index. -func (m TranslateOffsetMap) SetIndexOffset(name string, offset uint64) { +func (m TranslateOffsetMap) SetIndexPartitionOffset(name string, partitionID int, offset uint64) { if m[name] == nil { - m[name] = make(map[string]uint64) + m[name] = NewIndexTranslateOffsetMap() } - m[name][""] = offset + m[name].Partitions[partitionID] = offset } // FieldOffset returns the offset for the given field. @@ -177,15 +185,27 @@ func (m TranslateOffsetMap) FieldOffset(index, name string) uint64 { if m[index] == nil { return 0 } - return m[index][name] + return m[index].Fields[name] } // SetFieldOffset sets the offset for the given field. func (m TranslateOffsetMap) SetFieldOffset(index, name string, offset uint64) { if m[index] == nil { - m[index] = make(map[string]uint64) + m[index] = NewIndexTranslateOffsetMap() + } + m[index].Fields[name] = offset +} + +type IndexTranslateOffsetMap struct { + Partitions map[int]uint64 `json:"partitions"` + Fields map[string]uint64 `json:"fields"` +} + +func NewIndexTranslateOffsetMap() *IndexTranslateOffsetMap { + return &IndexTranslateOffsetMap{ + Partitions: make(map[int]uint64), + Fields: make(map[string]uint64), } - m[index][name] = offset } // Ensure type implements interface. @@ -193,19 +213,21 @@ var _ TranslateStore = &InMemTranslateStore{} // InMemTranslateStore is an in-memory storage engine for mapping keys to int values. type InMemTranslateStore struct { - mu sync.RWMutex - index string - field string - readOnly bool - keys []string - lookup map[string]uint64 + mu sync.RWMutex + partitionID int + index string + field string + readOnly bool + keys []string + lookup map[string]uint64 writeNotify chan struct{} } // NewInMemTranslateStore returns a new instance of InMemTranslateStore. -func NewInMemTranslateStore(index, field string) *InMemTranslateStore { +func NewInMemTranslateStore(index, field string, partitionID int) *InMemTranslateStore { return &InMemTranslateStore{ + partitionID: partitionID, index: index, field: field, lookup: make(map[string]uint64), @@ -217,14 +239,19 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. // Implements OpenTranslateStoreFunc. -func OpenInMemTranslateStore(rawurl, index, field string) (TranslateStore, error) { - return NewInMemTranslateStore(index, field), nil +func OpenInMemTranslateStore(rawurl, index, field string, partitionID int) (TranslateStore, error) { + return NewInMemTranslateStore(index, field, partitionID), nil } func (s *InMemTranslateStore) Close() error { return nil } +// PartitionID returns the partition id the store was initialized with. +func (s *InMemTranslateStore) PartitionID() int { + return s.partitionID +} + // ReadOnly returns true if the store is in read-only mode. func (s *InMemTranslateStore) ReadOnly() bool { s.mu.Lock() diff --git a/translator_test.go b/translator_test.go index 664a18fba..e8794d2fe 100644 --- a/translator_test.go +++ b/translator_test.go @@ -26,7 +26,7 @@ import ( ) func TestInMemTranslateStore_TranslateKey(t *testing.T) { - s := pilosa.NewInMemTranslateStore("IDX", "FLD") + s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0) // Ensure initial key translates to ID 1. if id, err := s.TranslateKey("foo"); err != nil { @@ -51,7 +51,7 @@ func TestInMemTranslateStore_TranslateKey(t *testing.T) { } func TestInMemTranslateStore_TranslateID(t *testing.T) { - s := pilosa.NewInMemTranslateStore("IDX", "FLD") + s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0) // Setup initial keys. if _, err := s.TranslateKey("foo"); err != nil { From b3e86e839492b40a94b36ce68be50de5b6d54495 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 14 Dec 2019 14:11:59 -0700 Subject: [PATCH 03/27] refactoring stores back into index/field --- api.go | 104 +++++++++++- cluster.go | 341 ++++++++++++++++---------------------- encoding/proto/proto.go | 20 +++ executor.go | 12 +- executor_internal_test.go | 4 - field.go | 17 ++ holder.go | 4 + http/client.go | 2 +- http/handler.go | 25 +++ index.go | 26 +++ server.go | 4 +- server/grpc.go | 4 +- 12 files changed, 342 insertions(+), 221 deletions(-) diff --git a/api.go b/api.go index 03c3213e8..381387dbb 100644 --- a/api.go +++ b/api.go @@ -545,7 +545,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin var err error if field.keys() { - if rowStr, err = api.cluster.translateFieldID(indexName, fieldName, rowID); err != nil { + if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil { return errors.Wrap(err, "translating row") } } else { @@ -553,7 +553,9 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin } if index.Keys() { - if colStr, err = api.cluster.translateIndexPartitionID(ctx, indexName, api.cluster.idPartition(indexName, columnID), columnID); err != nil { + if store := index.TranslateStore(api.cluster.idPartition(indexName, columnID)); store == nil { + return errors.Wrap(err, "partition does not exist") + } else if colStr, err = store.TranslateID(columnID); err != nil { return errors.Wrap(err, "translating column") } } else { @@ -963,7 +965,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } - if req.RowIDs, err = api.cluster.translateFieldKeys(req.Index, req.Field, req.RowKeys); err != nil { + if req.RowIDs, err = field.TranslateStore().TranslateKeys(req.RowKeys); err != nil { return errors.Wrap(err, "translating rows") } } @@ -1368,10 +1370,67 @@ func (api *API) Info() serverInfo { } // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset. -func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (TranslateEntryReader, error) { +func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader") defer span.Finish() - return api.cluster.translateEntryReader(ctx, offsets) + + // Ensure all readers are cleaned up if any error. + var a []TranslateEntryReader + defer func() { + if err != nil { + for i := range a { + a[i].Close() + } + } + }() + + // Fetch all index partition readers. + for indexName, indexMap := range offsets { + index := api.holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + for partitionID, offset := range indexMap.Partitions { + store := index.TranslateStore(partitionID) + if store == nil { + return nil, ErrTranslateStoreNotFound + } + + r, err := store.EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "index partition translate reader") + } + a = append(a, r) + } + } + + // Fetch all field readers. + for indexName, indexMap := range offsets { + index := api.holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + for fieldName, offset := range indexMap.Fields { + field := index.Field(fieldName) + if field == nil { + return nil, ErrIndexNotFound + } + + r, err := field.TranslateStore().EntryReader(ctx, uint64(offset)) + if err != nil { + return nil, errors.Wrap(err, "field translate reader") + } + a = append(a, r) + } + } + + return NewMultiTranslateEntryReader(ctx, a), nil +} + +func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) { + return api.cluster.translateIndexKey(ctx, indexName, key) } // TranslateKeys handles a TranslateKeyRequest. @@ -1390,7 +1449,9 @@ func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err e return nil, err } } else { - if ids, err = api.cluster.translateFieldKeys(req.Index, req.Field, req.Keys); err != nil { + if field := api.holder.Field(req.Index, req.Field); field == nil { + return nil, ErrFieldNotFound + } else if ids, err = field.TranslateStore().TranslateKeys(req.Keys); err != nil { return nil, err } } @@ -1403,6 +1464,37 @@ func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err e return buf, nil } +// TranslateIDs handles a TranslateIDRequest. +func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) { + var req TranslateIDsRequest + if buf, err := ioutil.ReadAll(r); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error")) + } else if err := api.Serializer.Unmarshal(buf, &req); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error")) + } + + // Lookup store for either index or field and translate ids. + var keys []string + if req.Field == "" { + if keys, err = api.cluster.translateIndexIDs(ctx, req.Index, req.IDs); err != nil { + return nil, err + } + } else { + if field := api.holder.Field(req.Index, req.Field); field == nil { + return nil, ErrFieldNotFound + } else if keys, err = field.TranslateStore().TranslateIDs(req.IDs); err != nil { + return nil, err + } + } + + // Encode response. + buf, err := api.Serializer.Marshal(&TranslateIDsResponse{Keys: keys}) + if err != nil { + return nil, errors.Wrap(err, "translate ids response encoding error") + } + return buf, nil +} + // PrimaryReplicaNodeURL returns the URL of the cluster's primary replica. func (api *API) PrimaryReplicaNodeURL() url.URL { node := api.cluster.PrimaryReplicaNode() diff --git a/cluster.go b/cluster.go index 92516fb7c..f14ff96a3 100644 --- a/cluster.go +++ b/cluster.go @@ -215,10 +215,6 @@ type cluster struct { // nolint: maligned holder *Holder broadcaster broadcaster - // translation stores - indexTranslateStoreMap map[string]map[int]TranslateStore // store by index+partition - fieldTranslateStoreMap map[string]map[string]TranslateStore // store by index+field - joiningLeavingNodes chan nodeAction // joining is held open until this node @@ -240,9 +236,7 @@ type cluster struct { // nolint: maligned InternalClient InternalClient - // Instantiates new translation stores - OpenTranslateStore OpenTranslateStoreFunc - OpenTranslateReader OpenTranslateReaderFunc + // OpenTranslateReader OpenTranslateReaderFunc } // newCluster returns a new instance of Cluster with defaults. @@ -257,13 +251,8 @@ func newCluster() *cluster { closing: make(chan struct{}), joining: make(chan struct{}), - indexTranslateStoreMap: make(map[string]map[int]TranslateStore), - fieldTranslateStoreMap: make(map[string]map[string]TranslateStore), - InternalClient: newNopInternalClient(), - OpenTranslateStore: OpenInMemTranslateStore, - logger: logger.NopLogger, } } @@ -1011,10 +1000,6 @@ func (c *cluster) setup() error { if err != nil { return errors.Wrap(err, "adding local node") } - - if err := c.updateTranslateStores(); err != nil { - return err - } return nil } @@ -2024,11 +2009,6 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } - // Open appropriate translate stores. - if err := c.updateTranslateStores(); err != nil { - return err - } - c.unprotectedSetState(cs.State) c.markAsJoined() @@ -2085,6 +2065,145 @@ func (c *cluster) setStatic(hosts []string) error { return nil } +func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string) (uint64, error) { + keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}}) + if err != nil { + return 0, err + } + return keyMap[key], nil +} + +func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys []string) ([]uint64, error) { + keySet := make(map[string]struct{}) + for _, key := range keys { + keySet[key] = struct{}{} + } + + keyMap, err := c.translateIndexKeySet(ctx, indexName, keySet) + if err != nil { + return nil, err + } + + ids := make([]uint64, len(keys)) + for i := range keys { + ids[i] = keyMap[keys[i]] + } + return ids, nil +} + +func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}) (map[string]uint64, error) { + keyMap := make(map[string]uint64) + + idx := c.holder.Index(indexName) + if idx == nil { + return nil, ErrIndexNotFound + } + + // Split keys by partition. + keysByPartition := make(map[int][]string, c.partitionN) + for key := range keySet { + partitionID := c.keyPartition(indexName, key) + keysByPartition[partitionID] = append(keysByPartition[partitionID], key) + } + + // Translate keys by partition. + var g errgroup.Group + var mu sync.Mutex + for partitionID := range keysByPartition { + keys := keysByPartition[partitionID] + + g.Go(func() (err error) { + var ids []uint64 + if c.ownsPartition(c.Node.ID, partitionID) { + if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil { + return err + } + } else { + nodes := c.partitionNodes(partitionID) + if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, indexName, "", keys); err != nil { + return err + } + } + + mu.Lock() + defer mu.Unlock() + for i := range keys { + keyMap[keys[i]] = ids[i] + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return keyMap, nil +} + +func (c *cluster) translateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) { + idSet := make(map[uint64]struct{}) + for _, id := range ids { + idSet[id] = struct{}{} + } + + idMap, err := c.translateIndexIDSet(ctx, indexName, idSet) + if err != nil { + return nil, err + } + + keys := make([]string, len(ids)) + for i := range ids { + keys[i] = idMap[ids[i]] + } + return keys, nil +} + +func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idSet map[uint64]struct{}) (map[uint64]string, error) { + idMap := make(map[uint64]string) + + index := c.holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Split ids by partition. + idsByPartition := make(map[int][]uint64, c.partitionN) + for id := range idSet { + partitionID := c.idPartition(indexName, id) + idsByPartition[partitionID] = append(idsByPartition[partitionID], id) + } + + // Translate ids by partition. + var g errgroup.Group + var mu sync.Mutex + for partitionID, ids := range idsByPartition { + g.Go(func() (err error) { + var keys []string + if c.ownsPartition(c.Node.ID, partitionID) { + if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil { + return err + } + } else { + nodes := c.partitionNodes(partitionID) + if keys, err = c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, indexName, "", ids); err != nil { + return err + } + } + + mu.Lock() + defer mu.Unlock() + for i := range ids { + idMap[ids[i]] = keys[i] + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return idMap, nil +} + +/* // updateTranslateStores starts stores for partitions & fields owned by this // node and stops stores for ones this node does not own. func (c *cluster) updateTranslateStores() error { @@ -2115,23 +2234,6 @@ func (c *cluster) closeUnownedTranslateStores() error { } } - // Close index partition stores. - for indexName, m := range c.fieldTranslateStoreMap { - idx := c.holder.Index(indexName) - - for fieldName, store := range m { - if idx != nil && idx.Field(fieldName) != nil { - continue - } - if err := store.Close(); err != nil { - return err - } - delete(m, fieldName) - } - if len(c.fieldTranslateStoreMap) == 0 { - delete(c.fieldTranslateStoreMap, indexName) - } - } return nil } @@ -2157,26 +2259,6 @@ func (c *cluster) openOwnedTranslateStores() error { } } - // Open field stores. - for _, index := range c.holder.Indexes() { - m := c.fieldTranslateStoreMap[index.Name()] - if m == nil { - m = make(map[string]TranslateStore) - c.fieldTranslateStoreMap[index.Name()] = m - } - - for _, field := range index.Fields() { - if m[field.Name()] != nil { - continue - } - store, err := c.OpenTranslateStore(field.TranslateStorePath(), index.Name(), field.Name(), 0) - if err != nil { - return err - } - m[field.Name()] = store - } - } - return nil } @@ -2196,101 +2278,7 @@ func (c *cluster) fieldTranslateStore(index, field string) TranslateStore { return m[field] } -func (c *cluster) translateIndexKeys(ctx context.Context, index string, keys []string) ([]uint64, error) { - keySet := make(map[string]struct{}) - for _, key := range keys { - keySet[key] = struct{}{} - } - keyMap, err := c.translateIndexKeySet(ctx, index, keySet) - if err != nil { - return nil, err - } - - ids := make([]uint64, len(keys)) - for i := range keys { - ids[i] = keyMap[keys[i]] - } - return ids, nil -} - -func (c *cluster) translateIndexKeySet(ctx context.Context, index string, keySet map[string]struct{}) (map[string]uint64, error) { - keyMap := make(map[string]uint64) - - // Split keys by partition. - keysByPartition := make(map[int][]string, c.partitionN) - for key := range keySet { - partitionID := c.keyPartition(index, key) - keysByPartition[partitionID] = append(keysByPartition[partitionID], key) - } - - // Translate keys by partition. - var g errgroup.Group - var mu sync.Mutex - for partitionID := range keysByPartition { - keys := keysByPartition[partitionID] - - g.Go(func() (err error) { - var ids []uint64 - if c.ownsPartition(c.Node.ID, partitionID) { - if ids, err = c.translateIndexPartitionKeys(ctx, index, partitionID, keys); err != nil { - return err - } - } else { - nodes := c.partitionNodes(partitionID) - if ids, err = c.InternalClient.TranslateKeysNode(ctx, &nodes[0].URI, index, "", keys); err != nil { - return err - } - } - - mu.Lock() - defer mu.Unlock() - for i := range keys { - keyMap[keys[i]] = ids[i] - } - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - return keyMap, nil -} - -func (c *cluster) translateIndexIDSet(ctx context.Context, index string, idSet map[uint64]struct{}) (map[uint64]string, error) { - idMap := make(map[uint64]string) - - // Split ids by partition. - idsByPartition := make(map[int][]uint64, c.partitionN) - for id := range idSet { - partitionID := c.idPartition(index, id) - idsByPartition[partitionID] = append(idsByPartition[partitionID], id) - } - - // Translate ids by partition. - var g errgroup.Group - var mu sync.Mutex - for partitionID, ids := range idsByPartition { - g.Go(func() error { - nodes := c.partitionNodes(partitionID) - keys, err := c.InternalClient.TranslateIDsNode(ctx, &nodes[0].URI, index, "", ids) - if err != nil { - return err - } - - mu.Lock() - defer mu.Unlock() - for i := range ids { - idMap[ids[i]] = keys[i] - } - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - return idMap, nil -} func (c *cluster) translateIndexPartitionKeys(ctx context.Context, index string, partitionID int, keys []string) ([]uint64, error) { s := c.indexPartitionTranslateStore(index, partitionID) @@ -2388,54 +2376,7 @@ func (c *cluster) setTranslateStoreReadOnly(v bool) { } } } - -// translateEntryReader returns a reader that merges all index & field reader -// that are specified in the offsets map. -func (c *cluster) translateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { - // Ensure all readers are cleaned up if any error. - var a []TranslateEntryReader - defer func() { - if err != nil { - for i := range a { - a[i].Close() - } - } - }() - - // Fetch all index partition readers. - for indexName, indexMap := range offsets { - for partitionID, offset := range indexMap.Partitions { - store := c.indexPartitionTranslateStore(indexName, partitionID) - if store == nil { - return nil, ErrTranslateStoreNotFound - } - - r, err := store.EntryReader(ctx, uint64(offset)) - if err != nil { - return nil, errors.Wrap(err, "index partition translate reader") - } - a = append(a, r) - } - } - - // Fetch all field readers. - for indexName, indexMap := range offsets { - for fieldName, offset := range indexMap.Fields { - store := c.fieldTranslateStore(indexName, fieldName) - if store == nil { - return nil, ErrTranslateStoreNotFound - } - - r, err := store.EntryReader(ctx, uint64(offset)) - if err != nil { - return nil, errors.Wrap(err, "field translate reader") - } - a = append(a, r) - } - } - - return NewMultiTranslateEntryReader(ctx, a), nil -} +*/ /* // holderTranslateStoreReplicator manages the replication of translation store diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index d75b784e1..74313e4cf 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -273,6 +273,22 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeTranslateKeysResponse(msg, mt) return nil + case *pilosa.TranslateIDsRequest: + msg := &internal.TranslateIDsRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateIDsRequest") + } + decodeTranslateIDsRequest(msg, mt) + return nil + case *pilosa.TranslateIDsResponse: + msg := &internal.TranslateIDsResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateIDsResponse") + } + decodeTranslateIDsResponse(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -338,6 +354,10 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeTranslateKeysRequest(mt) case *pilosa.TranslateKeysResponse: return encodeTranslateKeysResponse(mt) + case *pilosa.TranslateIDsRequest: + return encodeTranslateIDsRequest(mt) + case *pilosa.TranslateIDsResponse: + return encodeTranslateIDsResponse(mt) } return nil } diff --git a/executor.go b/executor.go index ddec4e18b..a011dc5f8 100644 --- a/executor.go +++ b/executor.go @@ -3656,7 +3656,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call, keyMap m return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) } } else if value := callArgString(c, rowKey); value != "" { - id, err := e.Cluster.translateFieldKey(index, fieldName, value) + id, err := field.TranslateStore().TranslateKey(value) if err != nil { return err } @@ -3752,7 +3752,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call, k if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") } - id, err := e.Cluster.translateFieldKey(index, field.Name(), prevStr) + id, err := field.TranslateStore().TranslateKey(prevStr) if err != nil { return errors.Wrapf(err, "translating row key '%s'", prevStr) } @@ -3857,7 +3857,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { - key, err := field.translateStore.TranslateID(result.Pair.ID) + key, err := field.TranslateStore().TranslateID(result.Pair.ID) if err != nil { return nil, err } @@ -3881,7 +3881,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field.keys() { other := make([]Pair, len(result.Pairs)) for i := range result.Pairs { - key, err := field.translateStore.TranslateID(result.Pairs[i].ID) + key, err := field.TranslateStore().TranslateID(result.Pairs[i].ID) if err != nil { return nil, err } @@ -3908,7 +3908,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, ErrFieldNotFound } if field.keys() { - key, err := e.Cluster.translateFieldID(index, g.Field, g.RowID) + key, err := field.TranslateStore().TranslateID(g.RowID) if err != nil { return nil, errors.Wrap(err, "translating row ID in Group") } @@ -3939,7 +3939,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } else if field.keys() { other.Keys = make([]string, len(result)) for i, id := range result { - key, err := e.Cluster.translateFieldID(index, fieldName, id) + key, err := field.TranslateStore().TranslateID(id) if err != nil { return nil, errors.Wrap(err, "translating row ID") } diff --git a/executor_internal_test.go b/executor_internal_test.go index 16588c1e2..c6ea6066a 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -54,10 +54,6 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("creating fields %v, %v, %v", erra, errb, errc) } - if err := cluster.updateTranslateStores(); err != nil { - t.Fatal(err) - } - query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"], having=Condition(count > 10))`) if err != nil { t.Fatalf("parsing query: %v", err) diff --git a/field.go b/field.go index 842be406d..41b1ed3b7 100644 --- a/field.go +++ b/field.go @@ -89,6 +89,11 @@ type Field struct { logger logger.Logger snapshotQueue snapshotQueue + + translateStore TranslateStore + + // Instantiates new translation stores + OpenTranslateStore OpenTranslateStoreFunc } // FieldOption is a functional option type for pilosa.fieldOptions. @@ -260,6 +265,8 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { remoteAvailableShards: roaring.NewBitmap(), logger: logger.NopLogger, + + OpenTranslateStore: OpenInMemTranslateStore, } return f, nil } @@ -278,6 +285,11 @@ func (f *Field) TranslateStorePath() string { return filepath.Join(f.path, "keys") } +// TranslateStore returns the field's translation store. +func (f *Field) TranslateStore() TranslateStore { + return f.translateStore +} + // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } @@ -468,6 +480,11 @@ func (f *Field) Open() error { return errors.Wrap(err, "opening attrstore") } + f.logger.Debugf("open translate store for index/field: %s/%s", f.index, f.name) + if f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, 0); err != nil { + return errors.Wrap(err, "opening field translate store") + } + return nil }(); err != nil { f.Close() diff --git a/holder.go b/holder.go index 2bc300ec1..d4abe0064 100644 --- a/holder.go +++ b/holder.go @@ -79,6 +79,10 @@ type Holder struct { // Manages replication from the primary node. primaryTranslateNode *Node + + // Instantiates new translation stores + OpenTranslateStore OpenTranslateStoreFunc + OpenTranslateReader OpenTranslateReaderFunc } // lockedChan looks a little ridiculous admittedly, but exists for good reason. diff --git a/http/client.go b/http/client.go index b99341c15..769d18d46 100644 --- a/http/client.go +++ b/http/client.go @@ -1183,7 +1183,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, return nil, pilosa.ErrIndexRequired } - buf, err := c.serializer.Marshal(pilosa.TranslateIDsRequest{ + buf, err := c.serializer.Marshal(&pilosa.TranslateIDsRequest{ Index: index, Field: field, IDs: ids, diff --git a/http/handler.go b/http/handler.go index 30d296a40..375fa09bb 100644 --- a/http/handler.go +++ b/http/handler.go @@ -309,6 +309,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff") router.HandleFunc("/internal/translate/data", handler.handlePostTranslateData).Methods("POST").Name("PostTranslateData") router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys") + router.HandleFunc("/internal/translate/ids", handler.handlePostTranslateIDs).Methods("POST").Name("PostTranslateIDs") router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff") router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") @@ -1757,6 +1758,30 @@ func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError) } + // Write response. + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing translate keys response: %v", err) + return + } +} + +func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + buf, err := h.api.TranslateIDs(r.Context(), r.Body) + if err != nil { + http.Error(w, fmt.Sprintf("translate ids: %v", err), http.StatusInternalServerError) + return + } + // Write response. _, err = w.Write(buf) if err != nil { diff --git a/index.go b/index.go index 57533b22a..29b5fc56b 100644 --- a/index.go +++ b/index.go @@ -61,6 +61,12 @@ type Index struct { // Used for notifying holder when a field is added. holder *Holder + + // Per-partition translation stores + translateStores map[int]TranslateStore + + // Instantiates new translation stores + OpenTranslateStore OpenTranslateStoreFunc } // NewIndex returns a new instance of Index. @@ -82,6 +88,10 @@ func NewIndex(path, name string) (*Index, error) { Stats: stats.NopStatsClient, logger: logger.NopLogger, trackExistence: true, + + translateStores: make(map[int]TranslateStore), + + OpenTranslateStore: OpenInMemTranslateStore, }, nil } @@ -96,6 +106,11 @@ func (i *Index) TranslateStorePath(partitionID int) string { return filepath.Join(i.path, "keys", strconv.Itoa(partitionID)) } +// TranslateStore returns the translation store for a given partition. +func (i *Index) TranslateStore(partitionID int) TranslateStore { + return i.translateStores[partitionID] +} + // Keys returns true if the index uses string keys. func (i *Index) Keys() bool { return i.keys } @@ -145,6 +160,16 @@ func (i *Index) Open() (err error) { return errors.Wrap(err, "opening attrstore") } + // TODO(BBJ): Support non-default partition counts. + i.logger.Debugf("open translate store for index: %s", i.name) + for partitionID := 0; partitionID < defaultPartitionN; partitionID++ { + store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID) + if err != nil { + return errors.Wrap(err, "opening index translate store") + } + i.translateStores[partitionID] = store + } + return nil } @@ -445,6 +470,7 @@ func (i *Index) newField(path, name string) (*Field, error) { if i.snapshotQueue != nil { f.snapshotQueue = i.snapshotQueue } + f.OpenTranslateStore = i.OpenTranslateStore return f, nil } diff --git a/server.go b/server.go index d89000bb9..96a4d5cde 100644 --- a/server.go +++ b/server.go @@ -284,7 +284,7 @@ func OptServerClusterHasher(h Hasher) ServerOption { // used to specify the translation data store type. func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption { return func(s *Server) error { - s.cluster.OpenTranslateStore = fn + s.holder.OpenTranslateStore = fn return nil } } @@ -293,7 +293,7 @@ func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption { // used to specify the remote translation data reader. func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { return func(s *Server) error { - s.cluster.OpenTranslateReader = fn + s.holder.OpenTranslateReader = fn return nil } } diff --git a/server/grpc.go b/server/grpc.go index a0030d6f5..7749fff1a 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -421,7 +421,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer case "int": // Translate column key. - id, err := index.TranslateStore().TranslateKey(col) + id, err := h.api.TranslateIndexKey(context.Background(), index.Name(), col) if err != nil { return errors.Wrap(err, "translating column key") } @@ -438,7 +438,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } case "decimal": // Translate column key. - id, err := index.TranslateStore().TranslateKey(col) + id, err := h.api.TranslateIndexKey(context.Background(), index.Name(), col) if err != nil { return errors.Wrap(err, "translating column key") } From f31d68739ed94c77bd5ae91b1be004cf8cbecae9 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 18 Dec 2019 08:38:59 -0700 Subject: [PATCH 04/27] holder syncer translate implementation --- holder.go | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 4 ++ 2 files changed, 185 insertions(+) diff --git a/holder.go b/holder.go index d4abe0064..91b939eb4 100644 --- a/holder.go +++ b/holder.go @@ -33,6 +33,7 @@ import ( "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" + "golang.org/x/sync/errgroup" ) const ( @@ -663,6 +664,9 @@ type holderSyncer struct { Node *Node Cluster *cluster + // Translation sync handling. + readers []TranslateEntryReader + // Stats Stats stats.StatsClient @@ -876,6 +880,183 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err return nil } +// BeginTranslationSync initializes streaming sync of translation data. +func (s *holderSyncer) BeginTranslationSync() error { + // Stop existing streams. + if err := s.stopTranslationSync(); err != nil { + return errors.Wrap(err, "stop translation sync") + } + + // Set read-only flag for all translation stores. + s.setTranslateReadOnlyFlags() + + // Connect to each node that has a primary for which we are a replica. + if err := s.initializeIndexTranslateReplication(); err != nil { + return errors.Wrap(err, "initialize index translate replication") + } + + // Connect to coordinator to stream field data. + if err := s.initializeFieldTranslateReplication(); err != nil { + return errors.Wrap(err, "initialize field translate replication") + } + return nil +} + +// stopTranslationSync closes and waits for all outstanding translation readers +// to complete. This should be called before reconnecting to the cluster in case +// of a cluster resize or schema change. +func (s *holderSyncer) stopTranslationSync() error { + var g errgroup.Group + for i := range s.readers { + rd := s.readers[i] + g.Go(func() error { + return rd.Close() + }) + } + return g.Wait() +} + +// setTranslateReadOnlyFlags updates all translation stores to enabled or disable +// writing new translation keys. Index stores are writable if the node owns the +// partition. Field stores are writable if the node is the coordinator. +func (s *holderSyncer) setTranslateReadOnlyFlags() { + isCoordinator := s.Cluster.isCoordinator() + + for _, index := range s.Holder.Indexes() { + for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { + ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID) + index.TranslateStore(partitionID).SetReadOnly(!ownsPartition) + } + + for _, field := range index.Fields() { + field.TranslateStore().SetReadOnly(!isCoordinator) + } + } +} + +// initializeIndexTranslateReplication connects to each node that is the +// primary for a partition that we are a replica of. +func (s *holderSyncer) initializeIndexTranslateReplication() error { + for _, node := range s.Cluster.Nodes() { + // Skip local node. + if node.ID == s.Node.ID { + continue + } + + // Build a map of partition offsets to stream from. + m := make(TranslateOffsetMap) + for _, index := range s.Holder.Indexes() { + for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { + partitionNodes := s.Cluster.partitionNodes(partitionID) + isPrimary := partitionNodes[0].ID == node.ID // remote is primary? + isReplica := Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica? + if !isPrimary || !isReplica { + continue + } + + store := index.TranslateStore(partitionID) + offset, err := store.MaxID() + if err != nil { + return errors.Wrapf(err, "cannot determine max id for %q", index.Name()) + } + m.SetIndexPartitionOffset(index.Name(), partitionID, offset) + } + } + + // Connect to remote not and begin streaming. + rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) + if err != nil { + return err + } + s.readers = append(s.readers, rd) + + go func() { defer rd.Close(); s.readIndexTranslateReader(rd) }() + } + + return nil +} + +// initializeFieldTranslateReplication connects the coordinator to stream field data. +func (s *holderSyncer) initializeFieldTranslateReplication() error { + // Skip if coordinator. + if s.Cluster.Node.IsCoordinator { + return nil + } + + // Build a map of partition offsets to stream from. + m := make(TranslateOffsetMap) + for _, index := range s.Holder.Indexes() { + for _, field := range index.Fields() { + store := field.TranslateStore() + offset, err := store.MaxID() + if err != nil { + return errors.Wrapf(err, "cannot determine max id for %q/%q", index.Name(), field.Name()) + } + m.SetFieldOffset(index.Name(), field.Name(), offset) + } + } + + // Connect to coordinator and begin streaming. + coordinator := s.Cluster.coordinatorNode() + rd, err := s.Holder.OpenTranslateReader(context.Background(), coordinator.URI.String(), m) + if err != nil { + return err + } + s.readers = append(s.readers, rd) + + go func() { defer rd.Close(); s.readFieldTranslateReader(rd) }() + + return nil +} + +func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) { + for { + var entry TranslateEntry + if err := rd.ReadEntry(&entry); err != nil { + s.Holder.Logger.Printf("cannot read index translate entry: %s", err) + return + } + + // Find appropriate store. + idx := s.Holder.Index(entry.Index) + if idx == nil { + s.Holder.Logger.Printf("index not found: %q", entry.Index) + return + } + + // Apply replication to store. + store := idx.TranslateStore(s.Cluster.keyPartition(entry.Index, entry.Key)) + if err := store.ForceSet(entry.ID, entry.Key); err != nil { + s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key) + return + } + } +} + +func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) { + for { + var entry TranslateEntry + if err := rd.ReadEntry(&entry); err != nil { + s.Holder.Logger.Printf("cannot read field translate entry: %s", err) + return + } + + // Find appropriate store. + f := s.Holder.Field(entry.Index, entry.Field) + if f == nil { + s.Holder.Logger.Printf("field not found: %q/%q", entry.Index, entry.Field) + return + } + + // Apply replication to store. + store := f.TranslateStore() + if err := store.ForceSet(entry.ID, entry.Key); err != nil { + s.Holder.Logger.Printf("cannot force set field translation data: %d=%q", entry.ID, entry.Key) + return + } + } +} + // holderCleaner removes fragments and data files that are no longer used. type holderCleaner struct { Node *Node diff --git a/server.go b/server.go index 96a4d5cde..47f55a85f 100644 --- a/server.go +++ b/server.go @@ -519,6 +519,10 @@ func (s *Server) Open() error { s.syncer.Closing = s.closing s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") + if err := s.syncer.BeginTranslationSync(); err != nil { + return err + } + // Start background monitoring. s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() From 82910911dd1700d3bdda921c9f24cc7f1e75e009 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 20 Dec 2019 16:18:04 -0700 Subject: [PATCH 05/27] refactoring id partitioning --- boltdb/translate.go | 28 ++++++++++---- cluster.go | 17 +++++--- cluster_internal_test.go | 2 +- executor.go | 40 ++++--------------- executor_internal_test.go | 2 +- field.go | 2 +- holder.go | 16 +++++--- holder_internal_test.go | 6 +-- index.go | 16 +++++--- index_internal_test.go | 2 +- index_test.go | 2 +- server.go | 20 ++++++++-- test/holder.go | 6 ++- test/index.go | 4 +- translate.go | 81 +++++++++++++++++++++++++++------------ translator_test.go | 4 +- utils_internal_test.go | 2 +- 17 files changed, 150 insertions(+), 100 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index dcfccc5b5..4168b1e14 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -32,8 +32,8 @@ var ( ) // OpenTranslateStore opens and initializes a boltdb translation store. -func OpenTranslateStore(path, index, field string, partitionID int) (pilosa.TranslateStore, error) { - s := NewTranslateStore(index, field, partitionID) +func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) { + s := NewTranslateStore(index, field, partitionID, partitionN) s.Path = path if err := s.Open(); err != nil { return nil, err @@ -52,6 +52,7 @@ type TranslateStore struct { index string field string partitionID int + partitionN int once sync.Once closing chan struct{} @@ -64,11 +65,12 @@ type TranslateStore struct { } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(index, field string, partitionID int) *TranslateStore { +func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore { return &TranslateStore{ index: index, field: field, partitionID: partitionID, + partitionN: partitionN, closing: make(chan struct{}), writeNotify: make(chan struct{}), } @@ -165,7 +167,9 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { bkt := tx.Bucket([]byte("keys")) if id = findIDByKey(bkt, key); id != 0 { return nil - } else if id, err = bkt.NextSequence(); err != nil { + } + + if id, err := pilosa.GenerateNextPartitionedID(maxID(tx), s.partitionID, s.partitionN); err != nil { return err } else if err := bkt.Put([]byte(key), u64tob(id)); err != nil { return err @@ -227,7 +231,9 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) { if ids[i] = findIDByKey(bkt, key); ids[i] != 0 { continue - } else if ids[i], err = bkt.NextSequence(); err != nil { + } + + if ids[i], err = pilosa.GenerateNextPartitionedID(maxID(tx), s.partitionID, s.partitionN); err != nil { return err } else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil { return err @@ -318,9 +324,7 @@ func (s *TranslateStore) notifyWrite() { // MaxID returns the highest id in the store. func (s *TranslateStore) MaxID() (max uint64, err error) { if err := s.db.View(func(tx *bolt.Tx) error { - if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil { - max = btou64(key) - } + max = maxID(tx) return nil }); err != nil { return 0, err @@ -328,6 +332,14 @@ func (s *TranslateStore) MaxID() (max uint64, err error) { return max, nil } +// MaxID returns the highest id in the store. +func maxID(tx *bolt.Tx) uint64 { + if key, _ := tx.Bucket([]byte("ids")).Cursor().Last(); key != nil { + return btou64(key) + } + return 0 +} + type TranslateEntryReader struct { ctx context.Context store *TranslateStore diff --git a/cluster.go b/cluster.go index f14ff96a3..05ba2a27c 100644 --- a/cluster.go +++ b/cluster.go @@ -40,8 +40,8 @@ import ( ) const ( - // defaultPartitionN is the default number of partitions in a cluster. - defaultPartitionN = 256 + // DefaultPartitionN is the default number of partitions in a cluster. + DefaultPartitionN = 256 // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" @@ -243,7 +243,7 @@ type cluster struct { // nolint: maligned func newCluster() *cluster { return &cluster{ Hasher: &jmphasher{}, - partitionN: defaultPartitionN, + partitionN: DefaultPartitionN, ReplicaN: 1, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel @@ -876,7 +876,7 @@ func (c *cluster) keyPartition(index, key string) int { // idPartition returns the partition that an id belongs to. func (c *cluster) idPartition(index string, id uint64) int { - return c.shardPartition(index, id/ShardWidth) + return c.shardPartition(index, (id/ShardWidth)%uint64(c.partitionN)) } // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -2110,10 +2110,12 @@ func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, ke var g errgroup.Group var mu sync.Mutex for partitionID := range keysByPartition { + partitionID := partitionID keys := keysByPartition[partitionID] g.Go(func() (err error) { var ids []uint64 + println("dbg/translateIndexKeySet", partitionID, len(keys), c.ownsPartition(c.Node.ID, partitionID)) if c.ownsPartition(c.Node.ID, partitionID) { if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil { return err @@ -2175,9 +2177,13 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS // Translate ids by partition. var g errgroup.Group var mu sync.Mutex - for partitionID, ids := range idsByPartition { + for partitionID := range idsByPartition { + partitionID := partitionID + ids := idsByPartition[partitionID] + g.Go(func() (err error) { var keys []string + println("dbg/translateIndexIDSet", partitionID, len(ids), c.ownsPartition(c.Node.ID, partitionID)) if c.ownsPartition(c.Node.ID, partitionID) { if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil { return err @@ -2192,6 +2198,7 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS mu.Lock() defer mu.Unlock() for i := range ids { + println("dbg/>>", ids[i], keys[i]) idMap[ids[i]] = keys[i] } return nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 97eb73fbd..de87ad5ae 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -96,7 +96,7 @@ func newIndexWithTempPath(name string) *Index { if err != nil { panic(err) } - index, err := NewIndex(path, name) + index, err := NewIndex(path, name, DefaultPartitionN) if err != nil { panic(err) } diff --git a/executor.go b/executor.go index a011dc5f8..a5fbb14d0 100644 --- a/executor.go +++ b/executor.go @@ -29,7 +29,6 @@ import ( "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" - "golang.org/x/sync/errgroup" ) // defaultField is the field used if one is not specified. @@ -3591,10 +3590,10 @@ func (e *executor) collectCallIndexKeys(index string, idx *Index, c *pql.Call, k } colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) - if c.Args[colKey] != nil && !isString(c.Args[colKey]) { - return errors.New("column value must be a string when index 'keys' option enabled") - } else if value := callArgString(c, colKey); value != "" { - keySet[value] = struct{}{} + if c.Args[colKey] != nil && isString(c.Args[colKey]) { + if value := callArgString(c, colKey); value != "" { + keySet[value] = struct{}{} + } } return nil } @@ -3768,6 +3767,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call, k } func (e *executor) translateResults(ctx context.Context, index string, idx *Index, calls []*pql.Call, results []interface{}) (err error) { + println("dbg/") span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults") defer span.Finish() @@ -3780,39 +3780,13 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde return err } } - - // Split ids by partition. - idsByPartition := make(map[int][]uint64, e.Cluster.partitionN) - for id := range idSet { - partitionID := e.Cluster.shardPartition(index, id/ShardWidth) - idsByPartition[partitionID] = append(idsByPartition[partitionID], id) - } - - // Translate ids by partition. - var g errgroup.Group - var mu sync.Mutex - for partitionID, ids := range idsByPartition { - g.Go(func() error { - nodes := e.Cluster.partitionNodes(partitionID) - keys, err := e.client.TranslateIDsNode(ctx, &nodes[0].URI, index, "", ids) - if err != nil { - return err - } - - mu.Lock() - defer mu.Unlock() - for i := range ids { - idMap[ids[i]] = keys[i] - } - return nil - }) - } - if err := g.Wait(); err != nil { + if idMap, err = e.Cluster.translateIndexIDSet(ctx, index, idSet); err != nil { return err } } for i := range results { + println("dbg/results.a", i) results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap) if err != nil { return err diff --git a/executor_internal_test.go b/executor_internal_test.go index c6ea6066a..7ccb26b15 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -25,7 +25,7 @@ import ( ) func TestExecutor_TranslateGroupByCall(t *testing.T) { - holder := NewHolder() + holder := NewHolder(DefaultPartitionN) cluster := newCluster() cluster.holder = holder diff --git a/field.go b/field.go index 41b1ed3b7..efd84933e 100644 --- a/field.go +++ b/field.go @@ -481,7 +481,7 @@ func (f *Field) Open() error { } f.logger.Debugf("open translate store for index/field: %s/%s", f.index, f.name) - if f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, 0); err != nil { + if f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1); err != nil { return errors.Wrap(err, "opening field translate store") } diff --git a/holder.go b/holder.go index 91b939eb4..c87cbe641 100644 --- a/holder.go +++ b/holder.go @@ -51,6 +51,9 @@ const ( type Holder struct { mu sync.RWMutex + // Partition count used by translation. + partitionN int + // Indexes by name. indexes map[string]*Index @@ -113,10 +116,11 @@ func (lc *lockedChan) Recv() { } // NewHolder returns a new instance of Holder. -func NewHolder() *Holder { +func NewHolder(partitionN int) *Holder { return &Holder{ - indexes: make(map[string]*Index), - closing: make(chan struct{}), + partitionN: partitionN, + indexes: make(map[string]*Index), + closing: make(chan struct{}), opened: lockedChan{ch: make(chan struct{})}, @@ -447,7 +451,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } func (h *Holder) newIndex(path, name string) (*Index, error) { - index, err := NewIndex(path, name) + index, err := NewIndex(path, name, h.partitionN) if err != nil { return nil, err } @@ -880,8 +884,8 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err return nil } -// BeginTranslationSync initializes streaming sync of translation data. -func (s *holderSyncer) BeginTranslationSync() error { +// ResetTranslationSync reinitializes streaming sync of translation data. +func (s *holderSyncer) ResetTranslationSync() error { // Stop existing streams. if err := s.stopTranslationSync(); err != nil { return errors.Wrap(err, "stop translation sync") diff --git a/holder_internal_test.go b/holder_internal_test.go index 062b3e808..afee39c6c 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -39,7 +39,7 @@ func (h *tHolder) Close() error { // Note that the holder must be Closed first. func (h *tHolder) Reopen() error { path, logger := h.Path, h.Holder.Logger - h.Holder = NewHolder() + h.Holder = NewHolder(DefaultPartitionN) h.Holder.Path = path h.Holder.Logger = logger return h.Holder.Open() @@ -51,7 +51,7 @@ func newHolder() *tHolder { panic(err) } - h := &tHolder{Holder: NewHolder()} + h := &tHolder{Holder: NewHolder(DefaultPartitionN)} h.Path = path return h } @@ -302,7 +302,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Ensure holder can reopen. func TestHolderCleaner_Reopen(t *testing.T) { - h := NewHolder() + h := NewHolder(DefaultPartitionN) h.Path = "path" err := h.Open() if err != nil { diff --git a/index.go b/index.go index 29b5fc56b..bcf63f1f1 100644 --- a/index.go +++ b/index.go @@ -45,6 +45,9 @@ type Index struct { trackExistence bool existenceFld *Field + // Partitions used by translation. + partitionN int + // Fields by name. fields map[string]*Field @@ -70,16 +73,17 @@ type Index struct { } // NewIndex returns a new instance of Index. -func NewIndex(path, name string) (*Index, error) { +func NewIndex(path, name string, partitionN int) (*Index, error) { err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } return &Index{ - path: path, - name: name, - fields: make(map[string]*Field), + path: path, + name: name, + partitionN: partitionN, + fields: make(map[string]*Field), newAttrStore: newNopAttrStore, columnAttrs: nopStore, @@ -162,8 +166,8 @@ func (i *Index) Open() (err error) { // TODO(BBJ): Support non-default partition counts. i.logger.Debugf("open translate store for index: %s", i.name) - for partitionID := 0; partitionID < defaultPartitionN; partitionID++ { - store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID) + for partitionID := 0; partitionID < i.partitionN; partitionID++ { + store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.partitionN) if err != nil { return errors.Wrap(err, "opening index translate store") } diff --git a/index_internal_test.go b/index_internal_test.go index dc0296049..026607da4 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -25,7 +25,7 @@ func mustOpenIndex(opt IndexOptions) *Index { if err != nil { panic(err) } - index, err := NewIndex(path, "i") + index, err := NewIndex(path, "i", DefaultPartitionN) if err != nil { panic(err) } diff --git a/index_test.go b/index_test.go index 13084e322..77736331c 100644 --- a/index_test.go +++ b/index_test.go @@ -217,7 +217,7 @@ func TestIndex_InvalidName(t *testing.T) { if err != nil { panic(err) } - index, err := pilosa.NewIndex(path, "ABC") + index, err := pilosa.NewIndex(path, "ABC", pilosa.DefaultPartitionN) if err == nil { t.Fatalf("should have gotten an error on index name with caps") } diff --git a/server.go b/server.go index 47f55a85f..722ad0488 100644 --- a/server.go +++ b/server.go @@ -300,10 +300,12 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { + cluster := newCluster() + s := &Server{ closing: make(chan struct{}), - cluster: newCluster(), - holder: NewHolder(), + cluster: cluster, + holder: NewHolder(cluster.partitionN), diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), systemInfo: newNopSystemInfo(), defaultClient: nopInternalClient{}, @@ -519,7 +521,7 @@ func (s *Server) Open() error { s.syncer.Closing = s.closing s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") - if err := s.syncer.BeginTranslationSync(); err != nil { + if err := s.syncer.ResetTranslationSync(); err != nil { return err } @@ -656,10 +658,16 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if err := s.syncer.ResetTranslationSync(); err != nil { + return err + } case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err } + if err := s.syncer.ResetTranslationSync(); err != nil { + return err + } case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { @@ -670,11 +678,17 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if err := s.syncer.ResetTranslationSync(); err != nil { + return err + } case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { return err } + if err := s.syncer.ResetTranslationSync(); err != nil { + return err + } case *DeleteAvailableShardMessage: f := s.holder.Field(obj.Index, obj.Field) if err := f.RemoveAvailableShard(obj.ShardID); err != nil { diff --git a/test/holder.go b/test/holder.go index b5e7c1260..05182dfd4 100644 --- a/test/holder.go +++ b/test/holder.go @@ -23,6 +23,8 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" ) +const PartitionN = 256 + // Holder is a test wrapper for pilosa.Holder. type Holder struct { *pilosa.Holder @@ -35,7 +37,7 @@ func NewHolder() *Holder { panic(err) } - h := &Holder{Holder: pilosa.NewHolder()} + h := &Holder{Holder: pilosa.NewHolder(PartitionN)} h.Path = path h.Holder.NewAttrStore = boltdb.NewAttrStore @@ -61,7 +63,7 @@ func (h *Holder) Close() error { // Note that the holder must be Closed first. func (h *Holder) Reopen() error { path, logger := h.Path, h.Holder.Logger - h.Holder = pilosa.NewHolder() + h.Holder = pilosa.NewHolder(PartitionN) h.Holder.Path = path h.Holder.Logger = logger h.Holder.NewAttrStore = boltdb.NewAttrStore diff --git a/test/index.go b/test/index.go index 7f461b927..4bd4e1305 100644 --- a/test/index.go +++ b/test/index.go @@ -32,7 +32,7 @@ func newIndex() *Index { if err != nil { panic(err) } - index, err := pilosa.NewIndex(path, "i") + index, err := pilosa.NewIndex(path, "i", PartitionN) if err != nil { panic(err) } @@ -62,7 +62,7 @@ func (i *Index) Reopen() error { } path, name := i.Path(), i.Name() - i.Index, err = pilosa.NewIndex(path, name) + i.Index, err = pilosa.NewIndex(path, name, PartitionN) if err != nil { return err } diff --git a/translate.go b/translate.go index cb799698a..1e3e2094c 100644 --- a/translate.go +++ b/translate.go @@ -16,6 +16,7 @@ package pilosa import ( "context" + "fmt" "io" "sync" @@ -66,7 +67,26 @@ type TranslateStore interface { } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. -type OpenTranslateStoreFunc func(path, index, field string, partitionID int) (TranslateStore, error) +type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error) + +// GenerateNextPartitionedID returns the next ID within the same partition. +func GenerateNextPartitionedID(prev uint64, partitionID, partitionN int) (uint64, error) { + // Generate the first available ID for partition if none previously existed. + var id uint64 + if prev == 0 { + return (uint64(partitionID) * ShardWidth), nil + } else if (prev/ShardWidth)%uint64(partitionN) != uint64(partitionID) { + return 0, fmt.Errorf("partition id mismatch: id=%d partition=%d", prev, partitionID) + } + + // Try to use the next ID if it is in the same partition. + if id++; (id/ShardWidth)%uint64(partitionN) == uint64(partitionID) { + return id, nil + } + + // Otherwise jump to the next possible shard that is in the same partition. + return (id - ShardWidth) + (ShardWidth * uint64(partitionN)), nil +} // TranslateEntryReader represents a stream of translation entries. type TranslateEntryReader interface { @@ -214,23 +234,27 @@ var _ TranslateStore = &InMemTranslateStore{} // InMemTranslateStore is an in-memory storage engine for mapping keys to int values. type InMemTranslateStore struct { mu sync.RWMutex - partitionID int index string field string + partitionID int + partitionN int readOnly bool - keys []string - lookup map[string]uint64 + keysByID map[uint64]string + idsByKey map[string]uint64 + maxID uint64 writeNotify chan struct{} } // NewInMemTranslateStore returns a new instance of InMemTranslateStore. -func NewInMemTranslateStore(index, field string, partitionID int) *InMemTranslateStore { +func NewInMemTranslateStore(index, field string, partitionID, partitionN int) *InMemTranslateStore { return &InMemTranslateStore{ - partitionID: partitionID, index: index, field: field, - lookup: make(map[string]uint64), + partitionID: partitionID, + partitionN: partitionN, + keysByID: make(map[uint64]string), + idsByKey: make(map[string]uint64), writeNotify: make(chan struct{}), } } @@ -239,8 +263,8 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. // Implements OpenTranslateStoreFunc. -func OpenInMemTranslateStore(rawurl, index, field string, partitionID int) (TranslateStore, error) { - return NewInMemTranslateStore(index, field, partitionID), nil +func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error) { + return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } func (s *InMemTranslateStore) Close() error { @@ -275,12 +299,12 @@ func (s *InMemTranslateStore) TranslateKey(key string) (uint64, error) { if s.readOnly { return 0, nil } - return s.translateKey(key), nil + return s.translateKey(key) } // TranslateKeys converts a string key to an integer ID. // If key does not have an associated id then one is created. -func (s *InMemTranslateStore) TranslateKeys(keys []string) ([]uint64, error) { +func (s *InMemTranslateStore) TranslateKeys(keys []string) (_ []uint64, err error) { s.mu.Lock() defer s.mu.Unlock() @@ -290,21 +314,30 @@ func (s *InMemTranslateStore) TranslateKeys(keys []string) ([]uint64, error) { ids := make([]uint64, len(keys)) for i := range keys { - ids[i] = s.translateKey(keys[i]) + if ids[i], err = s.translateKey(keys[i]); err != nil { + return ids, err + } } return ids, nil } -func (s *InMemTranslateStore) translateKey(key string) uint64 { +func (s *InMemTranslateStore) translateKey(key string) (_ uint64, err error) { // Return id if it has been added. - if id, ok := s.lookup[key]; ok { - return id + if id, ok := s.idsByKey[key]; ok { + return id, nil } // Generate a new id and update db. - id := uint64(len(s.keys) + 1) + var id uint64 + if s.field == "" { + if id, err = GenerateNextPartitionedID(s.maxID, s.partitionID, s.partitionN); err != nil { + return 0, err + } + } else { + id = s.maxID + 1 + } s.set(id, key) - return id + return id, nil } // TranslateID converts an integer ID to a string key. @@ -328,10 +361,7 @@ func (s *InMemTranslateStore) TranslateIDs(ids []uint64) ([]string, error) { } func (s *InMemTranslateStore) translateID(id uint64) string { - if id == 0 || id > uint64(len(s.keys)) { - return "" - } - return s.keys[id-1] + return s.keysByID[id] } // ForceSet writes the id/key pair to the db. Used by replication. @@ -344,8 +374,11 @@ func (s *InMemTranslateStore) ForceSet(id uint64, key string) error { // set assigns the id/key pair to the store. func (s *InMemTranslateStore) set(id uint64, key string) { - s.keys = append(s.keys, key) - s.lookup[key] = id + s.keysByID[id] = key + s.idsByKey[key] = id + if id > s.maxID { + s.maxID = id + } s.notifyWrite() } @@ -374,7 +407,7 @@ func (s *InMemTranslateStore) EntryReader(ctx context.Context, offset uint64) (T func (s *InMemTranslateStore) MaxID() (uint64, error) { s.mu.RLock() defer s.mu.RUnlock() - return uint64(len(s.keys)), nil + return s.maxID, nil } // inMemEntryReader represents a stream of translation entries for an inmem translation store. diff --git a/translator_test.go b/translator_test.go index e8794d2fe..34eaca0b1 100644 --- a/translator_test.go +++ b/translator_test.go @@ -26,7 +26,7 @@ import ( ) func TestInMemTranslateStore_TranslateKey(t *testing.T) { - s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0) + s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, pilosa.DefaultPartitionN) // Ensure initial key translates to ID 1. if id, err := s.TranslateKey("foo"); err != nil { @@ -51,7 +51,7 @@ func TestInMemTranslateStore_TranslateKey(t *testing.T) { } func TestInMemTranslateStore_TranslateID(t *testing.T) { - s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0) + s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, pilosa.DefaultPartitionN) // Setup initial keys. if _, err := s.TranslateKey("foo"); err != nil { diff --git a/utils_internal_test.go b/utils_internal_test.go index fe8e0d278..8abfdeb91 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -218,7 +218,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) } // holder - h := NewHolder() + h := NewHolder(DefaultPartitionN) h.Path = path // cluster From e3606d6615009c5f59649b6bf2596905b15382fe Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 21 Dec 2019 12:03:32 -0700 Subject: [PATCH 06/27] fix id generation --- api_test.go | 67 ++++----------------------------------------- boltdb/translate.go | 10 +++---- cluster.go | 17 ++++++++---- executor.go | 2 -- executor_test.go | 40 +++++++++++++-------------- field.go | 5 ++++ index.go | 8 ++++++ translate.go | 27 ++++++------------ 8 files changed, 62 insertions(+), 114 deletions(-) diff --git a/api_test.go b/api_test.go index b6cfa8d07..aabbaa700 100644 --- a/api_test.go +++ b/api_test.go @@ -203,14 +203,15 @@ func TestAPI_Import(t *testing.T) { // Generate some keyed records. rowIDs := []uint64{} - colKeys := []string{} timestamps := []int64{} for i := 1; i <= 10; i++ { rowIDs = append(rowIDs, rowID) timestamps = append(timestamps, timestamp) - colKeys = append(colKeys, fmt.Sprintf("col%d", i)) } + // Keys are sharded so ordering is not guaranteed. + colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ @@ -261,63 +262,6 @@ func TestAPI_Import(t *testing.T) { } }) - - t.Run("RowKeyColumnID", func(t *testing.T) { - ctx := context.Background() - index := "rkci" - field := "f" - - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100), pilosa.OptFieldKeys()) - if err != nil { - t.Fatalf("creating field: %v", err) - } - - rowKey := "rowkey" - - // Generate some keyed records. - rowKeys := []string{rowKey, rowKey, rowKey} - colIDs := []uint64{1, 2, pilosa.ShardWidth + 1} - timestamps := []int64{0, 0, 0} - - // Import data with keys to the coordinator (node0) and verify that it gets - // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) - req := &pilosa.ImportRequest{ - Index: index, - Field: field, - Shard: 0, - RowKeys: rowKeys, - ColumnIDs: colIDs, - Timestamps: timestamps, - } - if err := m0.API.Import(ctx, req); err != nil { - t.Fatal(err) - } - - pql := fmt.Sprintf("Row(%s=%s)", field, rowKey) - - // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - t.Fatalf("unexpected column ids: %+v", columns) - } - - // Query node1. - if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - return err - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, colIDs) { - return fmt.Errorf("unexpected column ids: %+v", columns) - } - return nil - }); err != nil { - t.Fatal(err) - } - }) } func TestAPI_ImportValue(t *testing.T) { @@ -356,12 +300,13 @@ func TestAPI_ImportValue(t *testing.T) { // Generate some keyed records. values := []int64{} - colKeys := []string{} for i := 1; i <= 10; i++ { values = append(values, int64(i)) - colKeys = append(colKeys, fmt.Sprintf("col%d", i)) } + // Column keys are sharded so their order is not guaranteed. + colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportValueRequest{ diff --git a/boltdb/translate.go b/boltdb/translate.go index 4168b1e14..dbdf5a867 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -169,9 +169,8 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { return nil } - if id, err := pilosa.GenerateNextPartitionedID(maxID(tx), s.partitionID, s.partitionN); err != nil { - return err - } else if err := bkt.Put([]byte(key), u64tob(id)); err != nil { + id := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + if err := bkt.Put([]byte(key), u64tob(id)); err != nil { return err } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { return err @@ -233,9 +232,8 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) { continue } - if ids[i], err = pilosa.GenerateNextPartitionedID(maxID(tx), s.partitionID, s.partitionN); err != nil { - return err - } else if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil { + ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil { return err } else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil { return err diff --git a/cluster.go b/cluster.go index 05ba2a27c..fdbf771ea 100644 --- a/cluster.go +++ b/cluster.go @@ -855,6 +855,10 @@ func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSour // shardPartition returns the partition that a shard belongs to. func (c *cluster) shardPartition(index string, shard uint64) int { + return shardPartition(index, shard, c.partitionN) +} + +func shardPartition(index string, shard uint64, partitionN int) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], shard) @@ -862,21 +866,25 @@ func (c *cluster) shardPartition(index string, shard uint64) int { h := fnv.New64a() _, _ = h.Write([]byte(index)) _, _ = h.Write(buf[:]) - return int(h.Sum64() % uint64(c.partitionN)) + return int(h.Sum64() % uint64(partitionN)) } // keyPartition returns the partition that a shard belongs to. func (c *cluster) keyPartition(index, key string) int { + return keyPartition(index, key, c.partitionN) +} + +func keyPartition(index, key string, partitionN int) int { // Hash the bytes and mod by partition count. h := fnv.New64a() _, _ = h.Write([]byte(index)) _, _ = h.Write([]byte(key)) - return int(h.Sum64() % uint64(c.partitionN)) + return int(h.Sum64() % uint64(partitionN)) } // idPartition returns the partition that an id belongs to. func (c *cluster) idPartition(index string, id uint64) int { - return c.shardPartition(index, (id/ShardWidth)%uint64(c.partitionN)) + return shardPartition(index, id/ShardWidth, c.partitionN) } // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -2115,7 +2123,6 @@ func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, ke g.Go(func() (err error) { var ids []uint64 - println("dbg/translateIndexKeySet", partitionID, len(keys), c.ownsPartition(c.Node.ID, partitionID)) if c.ownsPartition(c.Node.ID, partitionID) { if ids, err = idx.TranslateStore(partitionID).TranslateKeys(keys); err != nil { return err @@ -2183,7 +2190,6 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS g.Go(func() (err error) { var keys []string - println("dbg/translateIndexIDSet", partitionID, len(ids), c.ownsPartition(c.Node.ID, partitionID)) if c.ownsPartition(c.Node.ID, partitionID) { if keys, err = index.TranslateStore(partitionID).TranslateIDs(ids); err != nil { return err @@ -2198,7 +2204,6 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS mu.Lock() defer mu.Unlock() for i := range ids { - println("dbg/>>", ids[i], keys[i]) idMap[ids[i]] = keys[i] } return nil diff --git a/executor.go b/executor.go index a5fbb14d0..3f9a48d67 100644 --- a/executor.go +++ b/executor.go @@ -3767,7 +3767,6 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call, k } func (e *executor) translateResults(ctx context.Context, index string, idx *Index, calls []*pql.Call, results []interface{}) (err error) { - println("dbg/") span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults") defer span.Finish() @@ -3786,7 +3785,6 @@ func (e *executor) translateResults(ctx context.Context, index string, idx *Inde } for i := range results { - println("dbg/results.a", i) results[i], err = e.translateResult(index, idx, calls[i], results[i], idMap) if err != nil { return err diff --git a/executor_test.go b/executor_test.go index ec8865c8a..c2e679cbc 100644 --- a/executor_test.go +++ b/executor_test.go @@ -99,7 +99,7 @@ func TestExecutor_Execute_Row(t *testing.T) { readQueries := []string{`Row(f=1)`} responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}) - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two-hundred"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two-hundred", "one-hundred"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -324,7 +324,7 @@ func TestExecutor_Execute_Union(t *testing.T) { readQueries := []string{`Union(Row(f=10), Row(f=11))`} responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}) - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred", "one-hundred", "two"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -357,7 +357,7 @@ func TestExecutor_Execute_Union(t *testing.T) { responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, pilosa.OptFieldKeys()) - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred", "one-hundred", "two"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -1766,13 +1766,13 @@ func TestExecutor_Execute_Row_Range(t *testing.T) { pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) t.Run("Standard", func(t *testing.T) { - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "two", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) t.Run("Clear", func(t *testing.T) { - if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -1836,13 +1836,13 @@ func TestExecutor_Execute_Row_Range(t *testing.T) { pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "two", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) t.Run("Clear", func(t *testing.T) { - if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -1950,13 +1950,13 @@ func TestExecutor_Execute_Range_Deprecated(t *testing.T) { pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) t.Run("Standard", func(t *testing.T) { - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "two", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) t.Run("Clear", func(t *testing.T) { - if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -2020,13 +2020,13 @@ func TestExecutor_Execute_Range_Deprecated(t *testing.T) { pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "two", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) t.Run("Clear", func(t *testing.T) { - if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"four", "six", "seven", "five", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -2861,7 +2861,7 @@ func TestExecutor_Execute_Not(t *testing.T) { TrackExistence: true, Keys: true, }) - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"sw1", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -2892,7 +2892,7 @@ func TestExecutor_Execute_Not(t *testing.T) { TrackExistence: true, Keys: true, }, pilosa.OptFieldKeys()) - if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"sw1", "three"}) { t.Fatalf("unexpected keys: %+v", keys) } }) @@ -3016,12 +3016,12 @@ func TestExecutor_Execute_All(t *testing.T) { expCols []string expCnt uint64 }{ - {qry: "All()", expCols: req.ColumnKeys, expCnt: uint64(bitCount)}, - {qry: "All(limit=1)", expCols: req.ColumnKeys[:1], expCnt: 1}, - {qry: "All(limit=4)", expCols: req.ColumnKeys, expCnt: 4}, - {qry: "All(limit=5)", expCols: req.ColumnKeys, expCnt: 4}, - {qry: "All(limit=1, offset=1)", expCols: req.ColumnKeys[1:2], expCnt: 1}, - {qry: "All(limit=4, offset=1)", expCols: req.ColumnKeys[1:], expCnt: 3}, + {qry: "All()", expCols: []string{"c1", "c0", "c3", "c2"}, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: []string{"c1"}, expCnt: 1}, + {qry: "All(limit=4)", expCols: []string{"c1", "c0", "c3", "c2"}, expCnt: 4}, + {qry: "All(limit=5)", expCols: []string{"c1", "c0", "c3", "c2"}, expCnt: 4}, + {qry: "All(limit=1, offset=1)", expCols: []string{"c0"}, expCnt: 1}, + {qry: "All(limit=4, offset=1)", expCols: []string{"c0", "c3", "c2"}, expCnt: 3}, {qry: "All(limit=4, offset=5)", expCols: nil, expCnt: 0}, } for i, test := range tests { @@ -3034,7 +3034,7 @@ func TestExecutor_Execute_All(t *testing.T) { if len(cols) > 1000 || len(test.expCols) > 1000 { t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) } else { - t.Fatalf("test %d, unexpected columns, got: %T, but expected: %T", i, cols, test.expCols) + t.Fatalf("test %d, unexpected columns, got: %#v, but expected: %#v", i, cols, test.expCols) } } } diff --git a/field.go b/field.go index efd84933e..3fea5440d 100644 --- a/field.go +++ b/field.go @@ -725,6 +725,11 @@ func (f *Field) Close() error { _ = f.rowAttrStore.Close() } + // Close field translation store. + if err := f.translateStore.Close(); err != nil { + return err + } + // Close all views. for _, view := range f.viewMap { if err := view.close(); err != nil { diff --git a/index.go b/index.go index bcf63f1f1..a8adfc674 100644 --- a/index.go +++ b/index.go @@ -291,6 +291,14 @@ func (i *Index) Close() error { // Close the attribute store. i.columnAttrs.Close() + // Close partitioned translation stores. + for _, store := range i.translateStores { + if err := store.Close(); err != nil { + return errors.Wrap(err, "closing translation store") + } + } + i.translateStores = make(map[int]TranslateStore) + // Close all fields. for _, f := range i.fields { if err := f.Close(); err != nil { diff --git a/translate.go b/translate.go index 1e3e2094c..158677051 100644 --- a/translate.go +++ b/translate.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "fmt" "io" "sync" @@ -69,23 +68,15 @@ type TranslateStore interface { // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error) -// GenerateNextPartitionedID returns the next ID within the same partition. -func GenerateNextPartitionedID(prev uint64, partitionID, partitionN int) (uint64, error) { - // Generate the first available ID for partition if none previously existed. - var id uint64 - if prev == 0 { - return (uint64(partitionID) * ShardWidth), nil - } else if (prev/ShardWidth)%uint64(partitionN) != uint64(partitionID) { - return 0, fmt.Errorf("partition id mismatch: id=%d partition=%d", prev, partitionID) - } - +// GenerateNextPartitionedID returns the next ID within the same partition. +func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64 { // Try to use the next ID if it is in the same partition. - if id++; (id/ShardWidth)%uint64(partitionN) == uint64(partitionID) { - return id, nil + // Otherwise find ID in next shard that has a matching partition. + for id := prev + 1; ; id += ShardWidth { + if shardPartition(index, id/ShardWidth, partitionN) == partitionID { + return id + } } - - // Otherwise jump to the next possible shard that is in the same partition. - return (id - ShardWidth) + (ShardWidth * uint64(partitionN)), nil } // TranslateEntryReader represents a stream of translation entries. @@ -330,9 +321,7 @@ func (s *InMemTranslateStore) translateKey(key string) (_ uint64, err error) { // Generate a new id and update db. var id uint64 if s.field == "" { - if id, err = GenerateNextPartitionedID(s.maxID, s.partitionID, s.partitionN); err != nil { - return 0, err - } + id = GenerateNextPartitionedID(s.index, s.maxID, s.partitionID, s.partitionN) } else { id = s.maxID + 1 } From c0a129979ec9acbaf948e96ed9a32e743345b586 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 23 Dec 2019 15:53:39 -0700 Subject: [PATCH 07/27] fix tests --- executor_test.go | 2 +- field.go | 6 ++++-- pql/ast.go | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/executor_test.go b/executor_test.go index c2e679cbc..46ea95e8c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -127,7 +127,7 @@ func TestExecutor_Execute_Row(t *testing.T) { &pilosa.IndexOptions{Keys: true}, pilosa.OptFieldKeys()) if diff := cmp.Diff(responses[0].Results, []interface{}{ - &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + &pilosa.Row{Keys: []string{"bat", "foo"}, Attrs: map[string]interface{}{}}, }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { t.Fatal(diff) } diff --git a/field.go b/field.go index 3fea5440d..cd6b9c7ae 100644 --- a/field.go +++ b/field.go @@ -726,8 +726,10 @@ func (f *Field) Close() error { } // Close field translation store. - if err := f.translateStore.Close(); err != nil { - return err + if f.translateStore != nil { + if err := f.translateStore.Close(); err != nil { + return err + } } // Close all views. diff --git a/pql/ast.go b/pql/ast.go index d60b1062c..9d23c12b6 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -754,6 +754,8 @@ func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fiel return "", "_" + rowLabel, c.ArgString("_field") case "Rows": return "column", "previous", c.ArgString("_field") + case "IncludesColumn": + return "column", "", "" case "GroupBy": return "", "", "" default: From 9647d9b4bbb799d3c25a9dcc08daf608d1948c3b Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 27 Dec 2019 15:50:13 -0700 Subject: [PATCH 08/27] fixing additional tests --- boltdb/translate.go | 2 +- boltdb/translate_test.go | 48 ++++++++++++++++++++++------------------ ctl/import_test.go | 2 +- http/client_test.go | 48 ++++++++++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 41 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index dbdf5a867..0c67e74bc 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -169,7 +169,7 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { return nil } - id := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) if err := bkt.Put([]byte(key), u64tob(id)); err != nil { return err } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 7c3d0e3d5..219885324 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -28,24 +28,24 @@ func TestTranslateStore_TranslateKey(t *testing.T) { s := MustOpenNewTranslateStore() defer MustCloseTranslateStore(s) - // Ensure initial key translates to ID 1. + // Ensure initial key translates to first ID for shard if id, err := s.TranslateKey("foo"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(1); got != want { + } else if got, want := id, uint64(247463937); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } // Ensure next key autoincrements. if id, err := s.TranslateKey("bar"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(2); got != want { + } else if got, want := id, uint64(247463938); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } // Ensure retranslating existing key returns original ID. if id, err := s.TranslateKey("foo"); err != nil { t.Fatal(err) - } else if got, want := id, uint64(1); got != want { + } else if got, want := id, uint64(247463937); got != want { t.Fatalf("TranslateKey()=%d, want %d", got, want) } } @@ -57,29 +57,29 @@ func TestTranslateStore_TranslateKeys(t *testing.T) { // Ensure initial keys translate to incrementing IDs. if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { + } else if got, want := ids[1], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) } // Ensure retranslation returns original IDs. if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(2); got != want { + } else if got, want := ids[1], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) } // Ensure retranslating with existing and non-existing keys returns correctly. if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil { t.Fatal(err) - } else if got, want := ids[0], uint64(1); got != want { + } else if got, want := ids[0], uint64(247463937); got != want { t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want) - } else if got, want := ids[1], uint64(3); got != want { + } else if got, want := ids[1], uint64(247463939); got != want { t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want) - } else if got, want := ids[2], uint64(2); got != want { + } else if got, want := ids[2], uint64(247463938); got != want { t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want) } } @@ -89,20 +89,23 @@ func TestTranslateStore_TranslateID(t *testing.T) { defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKey("foo"); err != nil { + id1, err := s.TranslateKey("foo") + if err != nil { t.Fatal(err) - } else if _, err := s.TranslateKey("bar"); err != nil { + } + id2, err := s.TranslateKey("bar") + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if key, err := s.TranslateID(1); err != nil { + if key, err := s.TranslateID(id1); err != nil { t.Fatal(err) } else if got, want := key, "foo"; got != want { t.Fatalf("TranslateID()=%s, want %s", got, want) } - if key, err := s.TranslateID(2); err != nil { + if key, err := s.TranslateID(id2); err != nil { t.Fatal(err) } else if got, want := key, "bar"; got != want { t.Fatalf("TranslateID()=%s, want %s", got, want) @@ -114,12 +117,13 @@ func TestTranslateStore_TranslateIDs(t *testing.T) { defer MustCloseTranslateStore(s) // Setup initial keys. - if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil { + ids, err := s.TranslateKeys([]string{"foo", "bar"}) + if err != nil { t.Fatal(err) } // Ensure IDs can be translated back to keys. - if keys, err := s.TranslateIDs([]uint64{1, 2, 3}); err != nil { + if keys, err := s.TranslateIDs([]uint64{ids[0], ids[1], 1}); err != nil { t.Fatal(err) } else if got, want := keys[0], "foo"; got != want { t.Fatalf("TranslateIDs()[0]=%s, want %s", got, want) @@ -151,7 +155,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read first entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, uint64(247463937); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -160,7 +164,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read next entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(2); got != want { + } else if got, want := entry.ID, uint64(247463938); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "bar"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -174,7 +178,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { // Read newly created entry. if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(3); got != want { + } else if got, want := entry.ID, uint64(247463939); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "baz"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -211,7 +215,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { var entry pilosa.TranslateEntry if err := r.ReadEntry(&entry); err != nil { t.Fatal(err) - } else if got, want := entry.ID, uint64(1); got != want { + } else if got, want := entry.ID, uint64(247463937); got != want { t.Fatalf("ReadEntry() ID=%d, want %d", got, want) } else if got, want := entry.Key, "foo"; got != want { t.Fatalf("ReadEntry() Key=%s, want %s", got, want) @@ -302,7 +306,7 @@ func MustNewTranslateStore() *boltdb.TranslateStore { panic(err) } - s := boltdb.NewTranslateStore("I", "F") + s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN) s.Path = f.Name() return s } diff --git a/ctl/import_test.go b/ctl/import_test.go index ec8ed7666..62b9b3c33 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -310,7 +310,7 @@ func TestImportCommand_KeyReplication(t *testing.T) { if body, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatalf("reading: %s", err) } else if !reflect.DeepEqual(body, []byte(exp)) { - t.Fatalf("expected: %s, but got: %s", exp, body) + t.Errorf("expected: %s, but got: %s", exp, body) } } } diff --git a/http/client_test.go b/http/client_test.go index 077a11ff4..9405c0230 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -294,22 +294,29 @@ func TestClient_Export(t *testing.T) { buf := bytes.NewBuffer(nil) bw := bufio.NewWriter(buf) - // Send export request. - if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", 0, bw); err != nil { - t.Fatal(err) + // Send export request for every partition. + for i := 0; i < pilosa.DefaultPartitionN; i++ { + if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil { + t.Fatal(err) + } } got := buf.String() - // Expected output. - exp := "" - for _, bit := range data { - exp += fmt.Sprintf("%d,%s\n", bit.RowID, bit.ColumnKey) - } + // Expected output is not sorted because of key sharding. + exp := "" + + "2,col200\n" + + "2,col201\n" + + "2,col202\n" + + "2,col203\n" + + "1,col103\n" + + "1,col102\n" + + "1,col101\n" + + "1,col100\n" // Verify data. if got != exp { - t.Fatalf("unexpected export data: %s", got) + t.Fatalf("unexpected export data: %q, expected %q", got, exp) } }) @@ -329,21 +336,28 @@ func TestClient_Export(t *testing.T) { bw := bufio.NewWriter(buf) // Send export request. - if err := c.ExportCSV(context.Background(), "keyed", "keyedf", 0, bw); err != nil { - t.Fatal(err) + for i := 0; i < pilosa.DefaultPartitionN; i++ { + if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil { + t.Fatal(err) + } } got := buf.String() - // Expected output. - exp := "" - for _, bit := range data { - exp += fmt.Sprintf("%s,%s\n", bit.RowKey, bit.ColumnKey) - } + // Expected output is unsorted because of key sharding. + exp := "" + + "row2,col200\n" + + "row2,col201\n" + + "row2,col202\n" + + "row2,col203\n" + + "row1,col103\n" + + "row1,col102\n" + + "row1,col101\n" + + "row1,col100\n" // Verify data. if got != exp { - t.Fatalf("unexpected export data: %s", got) + t.Fatalf("unexpected export data: %q, expected %q", got, exp) } }) } From 2f76283f03d33c86fd30be3ceeaad78be198b095 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 28 Dec 2019 12:31:35 -0700 Subject: [PATCH 09/27] fix replication errors & test races --- ctl/import_test.go | 29 ++++++++++++++++++----------- holder.go | 17 +++++++++++++++++ server/handler_test.go | 2 +- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index 62b9b3c33..5c41a840c 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -25,6 +25,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/test" @@ -299,20 +300,26 @@ func TestImportCommand_KeyReplication(t *testing.T) { // Verify that the data is available on both nodes. for _, host := range []string{host0, host1} { - qry := "Count(Row(f=foo0))" - resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry))) - if err != nil { - t.Fatalf("Querying data for validation: %s", err) - } + if err := test.RetryUntil(2*time.Second, func() error { + qry := "Count(Row(f=foo0))" + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry))) + if err != nil { + return fmt.Errorf("Querying data for validation: %s", err) + } - // Read body and unmarshal response. - exp := `{"results":[100]}` + "\n" - if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatalf("reading: %s", err) - } else if !reflect.DeepEqual(body, []byte(exp)) { - t.Errorf("expected: %s, but got: %s", exp, body) + // Read body and unmarshal response. + exp := `{"results":[100]}` + "\n" + if body, err := ioutil.ReadAll(resp.Body); err != nil { + return fmt.Errorf("reading: %s", err) + } else if !reflect.DeepEqual(body, []byte(exp)) { + return fmt.Errorf("expected: %s, but got: %s", exp, body) + } + return nil + }); err != nil { + t.Fatal(err) } } + } // Ensure that integer import with keys runs. diff --git a/holder.go b/holder.go index c87cbe641..f4e61cc48 100644 --- a/holder.go +++ b/holder.go @@ -950,6 +950,10 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error { // Build a map of partition offsets to stream from. m := make(TranslateOffsetMap) for _, index := range s.Holder.Indexes() { + if !index.Keys() { + continue + } + for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { partitionNodes := s.Cluster.partitionNodes(partitionID) isPrimary := partitionNodes[0].ID == node.ID // remote is primary? @@ -967,6 +971,11 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error { } } + // Skip if no replication required. + if len(m) == 0 { + continue + } + // Connect to remote not and begin streaming. rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) if err != nil { @@ -990,6 +999,9 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error { // Build a map of partition offsets to stream from. m := make(TranslateOffsetMap) for _, index := range s.Holder.Indexes() { + if !index.Keys() { + continue + } for _, field := range index.Fields() { store := field.TranslateStore() offset, err := store.MaxID() @@ -1000,6 +1012,11 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error { } } + // Skip if no replication required. + if len(m) == 0 { + return nil + } + // Connect to coordinator and begin streaming. coordinator := s.Cluster.coordinatorNode() rd, err := s.Holder.OpenTranslateReader(context.Background(), coordinator.URI.String(), m) diff --git a/server/handler_test.go b/server/handler_test.go index 2f0af0345..cb132fe8d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -991,7 +991,7 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - target := []uint64{1, 2, 3} + target := []uint64{162529281, 159383553, 160432129} resp := pilosa.TranslateKeysResponse{} err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) if err != nil { From 1b068f75a889d98e253a07b4138a6a532239db0d Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 31 Dec 2019 08:49:32 -0700 Subject: [PATCH 10/27] Fix inmem read only translation bug --- translate.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/translate.go b/translate.go index 158677051..a883802ad 100644 --- a/translate.go +++ b/translate.go @@ -286,10 +286,6 @@ func (s *InMemTranslateStore) SetReadOnly(v bool) { func (s *InMemTranslateStore) TranslateKey(key string) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() - - if s.readOnly { - return 0, nil - } return s.translateKey(key) } @@ -299,10 +295,6 @@ func (s *InMemTranslateStore) TranslateKeys(keys []string) (_ []uint64, err erro s.mu.Lock() defer s.mu.Unlock() - if s.readOnly { - return nil, nil - } - ids := make([]uint64, len(keys)) for i := range keys { if ids[i], err = s.translateKey(keys[i]); err != nil { @@ -316,6 +308,8 @@ func (s *InMemTranslateStore) translateKey(key string) (_ uint64, err error) { // Return id if it has been added. if id, ok := s.idsByKey[key]; ok { return id, nil + } else if s.readOnly { + return 0, nil } // Generate a new id and update db. From 1409ab56645b490123aa51d9e1d26bfdd04030be Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 31 Dec 2019 09:21:36 -0700 Subject: [PATCH 11/27] fix data race --- field.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/field.go b/field.go index cd6b9c7ae..f579bef4a 100644 --- a/field.go +++ b/field.go @@ -1490,7 +1490,10 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO requiredDepth = v } // Increase bit depth if required. - if requiredDepth > bsig.BitDepth { + f.mu.RLock() + bitDepth := bsig.BitDepth + f.mu.RUnlock() + if requiredDepth > bitDepth { if err := func() error { f.mu.Lock() defer f.mu.Unlock() @@ -1501,7 +1504,7 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return errors.Wrap(err, "increasing bsi bit depth") } } else { - requiredDepth = bsig.BitDepth + requiredDepth = bitDepth } // Import into each fragment. From a189477ba3fcc1c4dc945dae37f4e35d9ad9cc55 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 2 Jan 2020 09:07:56 -0700 Subject: [PATCH 12/27] rebase & fix test const --- cluster.go | 274 ------------------------------------------------- test/holder.go | 6 +- test/index.go | 4 +- 3 files changed, 4 insertions(+), 280 deletions(-) diff --git a/cluster.go b/cluster.go index fdbf771ea..746a8e4c9 100644 --- a/cluster.go +++ b/cluster.go @@ -2215,280 +2215,6 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS return idMap, nil } -/* -// updateTranslateStores starts stores for partitions & fields owned by this -// node and stops stores for ones this node does not own. -func (c *cluster) updateTranslateStores() error { - if err := c.closeUnownedTranslateStores(); err != nil { - return err - } else if err := c.openOwnedTranslateStores(); err != nil { - return err - } - return nil -} - -func (c *cluster) closeUnownedTranslateStores() error { - for indexName, m := range c.indexTranslateStoreMap { - idx := c.holder.Index(indexName) - - // Close unowned partition stores. - for partitionID, store := range m { - if idx != nil && c.ownsShard(c.Node.ID, indexName, uint64(partitionID)) { - continue - } - if err := store.Close(); err != nil { - return err - } - delete(m, partitionID) - } - if len(c.indexTranslateStoreMap) == 0 { - delete(c.indexTranslateStoreMap, indexName) - } - } - - return nil -} - -// openOwnedTranslateStores ensures that all owned partition & field stores are open. -func (c *cluster) openOwnedTranslateStores() error { - // Open partition stores. - for _, index := range c.holder.Indexes() { - m := c.indexTranslateStoreMap[index.Name()] - if m == nil { - m = make(map[int]TranslateStore) - c.indexTranslateStoreMap[index.Name()] = m - } - - for partitionID := 0; partitionID < c.partitionN; partitionID++ { - if m[partitionID] != nil { - continue - } - store, err := c.OpenTranslateStore(index.TranslateStorePath(partitionID), index.Name(), "", partitionID) - if err != nil { - return err - } - m[partitionID] = store - } - } - - return nil -} - -func (c *cluster) indexPartitionTranslateStore(index string, partitionID int) TranslateStore { - m := c.indexTranslateStoreMap[index] - if m == nil { - return nil - } - return m[partitionID] -} - -func (c *cluster) fieldTranslateStore(index, field string) TranslateStore { - m := c.fieldTranslateStoreMap[index] - if m == nil { - return nil - } - return m[field] -} - - - -func (c *cluster) translateIndexPartitionKeys(ctx context.Context, index string, partitionID int, keys []string) ([]uint64, error) { - s := c.indexPartitionTranslateStore(index, partitionID) - if s == nil { - return nil, ErrTranslateStoreNotFound - } - return s.TranslateKeys(keys) -} - -func (c *cluster) translateIndexPartitionIDs(ctx context.Context, index string, partitionID int, ids []uint64) ([]string, error) { - s := c.indexPartitionTranslateStore(index, partitionID) - if s == nil { - return nil, ErrTranslateStoreNotFound - } - return s.TranslateIDs(ids) -} - -func (c *cluster) translateIndexPartitionID(ctx context.Context, index string, partitionID int, id uint64) (string, error) { - s := c.indexPartitionTranslateStore(index, partitionID) - if s == nil { - return "", ErrTranslateStoreNotFound - } - return s.TranslateID(id) -} - -func (c *cluster) translateFieldKey(index, field string, key string) (uint64, error) { - s := c.fieldTranslateStore(index, field) - if s == nil { - return 0, ErrTranslateStoreNotFound - } - return s.TranslateKey(key) -} - -func (c *cluster) translateFieldKeys(index, field string, keys []string) ([]uint64, error) { - s := c.fieldTranslateStore(index, field) - if s == nil { - return nil, ErrTranslateStoreNotFound - } - return s.TranslateKeys(keys) -} - -func (c *cluster) translateFieldID(index, field string, id uint64) (string, error) { - s := c.fieldTranslateStore(index, field) - if s == nil { - return "", ErrTranslateStoreNotFound - } - return s.TranslateID(id) -} - -func (c *cluster) translateFieldIDs(index, field string, ids []uint64) ([]string, error) { - s := c.fieldTranslateStore(index, field) - if s == nil { - return nil, ErrTranslateStoreNotFound - } - return s.TranslateIDs(ids) -} - -// TranslateOffsetMap returns a map of offsets for all indexes & fields. -func (c *cluster) TranslateOffsetMap() (TranslateOffsetMap, error) { - m := make(TranslateOffsetMap) - - for index, partitionMap := range c.indexTranslateStoreMap { - for partitionID, store := range partitionMap { - id, err := store.MaxID() - if err != nil { - return nil, err - } - m.SetIndexPartitionOffset(index, partitionID, id+1) - } - } - - for index, fieldMap := range c.fieldTranslateStoreMap { - for field, store := range fieldMap { - id, err := store.MaxID() - if err != nil { - return nil, err - } - m.SetFieldOffset(index, field, id+1) - } - } - - return m, nil -} - -func (c *cluster) setTranslateStoreReadOnly(v bool) { - for _, partitionMap := range c.indexTranslateStoreMap { - for _, store := range partitionMap { - store.SetReadOnly(v) - } - } - - for _, fieldMap := range c.fieldTranslateStoreMap { - for _, store := range fieldMap { - store.SetReadOnly(v) - } - } -} -*/ - -/* -// holderTranslateStoreReplicator manages the replication of translation store -// data from a primary store to the local replica. Continually tries to -// reconnect on disconnect. -type holderTranslateStoreReplicator struct { - ctx context.Context - cancel func() - wg sync.WaitGroup - - holder *Holder - nodeURL string - - logger logger.Logger -} - -func newHolderTranslateStoreReplicator(h *Holder, nodeURL string) *holderTranslateStoreReplicator { - r := &holderTranslateStoreReplicator{ - holder: h, - nodeURL: nodeURL, - logger: logger.NopLogger, - } - r.ctx, r.cancel = context.WithCancel(context.Background()) - return r -} - -// Open starts the background monitoring goroutine. -func (r *holderTranslateStoreReplicator) Open() error { - r.wg.Add(1) - go func() { defer r.wg.Done(); r.monitor() }() - return nil -} - -// Close stops the replicator. -func (r *holderTranslateStoreReplicator) Close() error { - r.cancel() - return nil -} - -// monitor runs in a background goroutine and continually tries to connect and -// stream translate changes from the primary store. -func (r *holderTranslateStoreReplicator) monitor() { - for { - select { - case <-r.ctx.Done(): - return - default: - if err := r.replicate(); err != nil { - r.logger.Printf("cannot replicate: nodeURL=%s err=%s", r.nodeURL, err) - } - time.Sleep(1 * time.Second) - } - } -} - -func (r *holderTranslateStoreReplicator) replicate() error { - // Determine the offsets of every index & field store. - offsets, err := r.holder.TranslateOffsetMap() - if err != nil { - return err - } else if len(offsets) == 0 { - return nil - } - - // Begin streaming from remote primary. - rd, err := r.holder.OpenTranslateReader(r.ctx, r.nodeURL, offsets) - if err != nil { - return err - } - defer rd.Close() - - for { - var entry TranslateEntry - if err := rd.ReadEntry(&entry); err != nil { - return err - } - - // Find appropriate store. - var store TranslateStore - if entry.Field == "" { - idx := r.holder.Index(entry.Index) - if idx == nil { - return ErrIndexNotFound - } - store = idx.TranslateStore() - } else { - f := r.holder.Field(entry.Index, entry.Field) - if f == nil { - return ErrFieldNotFound - } - store = f.TranslateStore() - } - - // Apply replication to store. - if err := store.ForceSet(entry.ID, entry.Key); err != nil { - return err - } - } -} -*/ // ClusterStatus describes the status of the cluster including its // state and node topology. diff --git a/test/holder.go b/test/holder.go index 05182dfd4..a4399ba67 100644 --- a/test/holder.go +++ b/test/holder.go @@ -23,8 +23,6 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" ) -const PartitionN = 256 - // Holder is a test wrapper for pilosa.Holder. type Holder struct { *pilosa.Holder @@ -37,7 +35,7 @@ func NewHolder() *Holder { panic(err) } - h := &Holder{Holder: pilosa.NewHolder(PartitionN)} + h := &Holder{Holder: pilosa.NewHolder(pilosa.DefaultPartitionN)} h.Path = path h.Holder.NewAttrStore = boltdb.NewAttrStore @@ -63,7 +61,7 @@ func (h *Holder) Close() error { // Note that the holder must be Closed first. func (h *Holder) Reopen() error { path, logger := h.Path, h.Holder.Logger - h.Holder = pilosa.NewHolder(PartitionN) + h.Holder = pilosa.NewHolder(pilosa.DefaultPartitionN) h.Holder.Path = path h.Holder.Logger = logger h.Holder.NewAttrStore = boltdb.NewAttrStore diff --git a/test/index.go b/test/index.go index 4bd4e1305..bf65c6ae2 100644 --- a/test/index.go +++ b/test/index.go @@ -32,7 +32,7 @@ func newIndex() *Index { if err != nil { panic(err) } - index, err := pilosa.NewIndex(path, "i", PartitionN) + index, err := pilosa.NewIndex(path, "i", pilosa.DefaultPartitionN) if err != nil { panic(err) } @@ -62,7 +62,7 @@ func (i *Index) Reopen() error { } path, name := i.Path(), i.Name() - i.Index, err = pilosa.NewIndex(path, name, PartitionN) + i.Index, err = pilosa.NewIndex(path, name, pilosa.DefaultPartitionN) if err != nil { return err } From bdfdeb1291ba5a309043268422694b8e2ba1386c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 2 Jan 2020 09:19:35 -0700 Subject: [PATCH 13/27] fix lint --- executor_internal_test.go | 4 +++- holder.go | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/executor_internal_test.go b/executor_internal_test.go index 7ccb26b15..b4487dca6 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -30,7 +30,9 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { cluster := newCluster() cluster.holder = holder cluster.Node = &Node{ID: "node1", URI: NewTestURIFromHostPort("node1", 0)} - cluster.addNode(cluster.Node) + if err := cluster.addNode(cluster.Node); err != nil { + t.Fatal(err) + } e := &executor{ Holder: holder, diff --git a/holder.go b/holder.go index f4e61cc48..73e907be8 100644 --- a/holder.go +++ b/holder.go @@ -81,9 +81,6 @@ type Holder struct { snapshotQueue snapshotQueue - // Manages replication from the primary node. - primaryTranslateNode *Node - // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc OpenTranslateReader OpenTranslateReaderFunc From a043490996953349c27c0afce822a593e08ce6f2 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 3 Jan 2020 14:12:39 -0700 Subject: [PATCH 14/27] add multi-shard translation --- executor.go | 284 ++++++++++++++++++++------------------ executor_internal_test.go | 4 +- index.go | 1 - 3 files changed, 154 insertions(+), 135 deletions(-) diff --git a/executor.go b/executor.go index 3f9a48d67..125f46358 100644 --- a/executor.go +++ b/executor.go @@ -3526,164 +3526,196 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } } -func (e *executor) translateCalls(ctx context.Context, index string, idx *Index, calls []*pql.Call) (err error) { +func (e *executor) translateCalls(ctx context.Context, defaultIndexName string, defaultIdx *Index, calls []*pql.Call) (err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateCalls") defer span.Finish() - // TODO(BBJ): Handle cross-index boundaries. - keyMap := make(map[string]uint64) - if idx.Keys() { - // Collect all index keys. - keySet := make(map[string]struct{}) - for i := range calls { - if err := e.collectCallIndexKeys(index, idx, calls[i], keySet); err != nil { - return err - } - } - - if keyMap, err = e.Cluster.translateIndexKeySet(ctx, index, keySet); err != nil { + // Generate a list of all used + indexNameMap := make(map[string]struct{}) + for i := range calls { + if err := e.collectCallIndexNameMap(ctx, defaultIndexName, calls[i], indexNameMap); err != nil { return err } } - // Translate calls. - for i := range calls { - // Possibly change to another index for translation, if this - // call crosses index boundaries. - newIdxName := calls[i].CallIndex() - var newIdx *Index - if newIdxName == "" || newIdxName == index { - newIdxName = index - newIdx = idx - } else { - newIdx = idx.holder.indexes[newIdxName] - if newIdx == nil { - return fmt.Errorf("unknown index %q specified in cross-index call", newIdxName) + // Perform a separate batch translation for each separate index used. + for indexName := range indexNameMap { + // Determine the target index name. + if indexName == "" { + indexName = defaultIndexName + } + isDefaultIndex := indexName == defaultIndexName + + // Determine the target index. + idx := defaultIdx + if !isDefaultIndex { + idx = idx.holder.indexes[indexName] + if idx == nil { + return fmt.Errorf("unknown index %q specified in cross-index call", indexName) } } - if err := e.translateCall(newIdxName, newIdx, calls[i], keyMap); err != nil { + + // Collect all index keys & bulk translate them. + keyMap := make(map[string]uint64) + if idx.Keys() { + keySet := make(map[string]struct{}) + for i := range calls { + if err := e.collectCallIndexKeys(indexName, idx, isDefaultIndex, calls[i], keySet); err != nil { + return err + } + } + + if keyMap, err = e.Cluster.translateIndexKeySet(ctx, indexName, keySet); err != nil { + return err + } + } + + // Translate calls. + for i := range calls { + if err := e.translateCall(indexName, idx, isDefaultIndex, calls[i], keyMap); err != nil { + return err + } + } + } + return nil +} + +func (e *executor) collectCallIndexNameMap(ctx context.Context, defaultIndexName string, c *pql.Call, m map[string]struct{}) error { + callIndex := c.CallIndex() + if callIndex == "" { + callIndex = defaultIndexName + } + m[callIndex] = struct{}{} + + if c.Name == "GroupBy" { + if filter, ok, err := c.CallArg("filter"); ok { + if err != nil { + return errors.Wrap(err, "getting filter call") + } + err := e.collectCallIndexNameMap(ctx, defaultIndexName, filter, m) + if err != nil { + return errors.Wrap(err, "collecting filter call index name") + } + } + } + + for _, child := range c.Children { + if err := e.collectCallIndexNameMap(ctx, defaultIndexName, child, m); err != nil { return err } } return nil } -func (e *executor) collectCallIndexKeys(index string, idx *Index, c *pql.Call, keySet map[string]struct{}) error { +func (e *executor) collectCallIndexKeys(index string, idx *Index, isDefaultIndex bool, c *pql.Call, keySet map[string]struct{}) error { // Handle group by separately. if c.Name == "GroupBy" { for _, child := range c.Children { - if err := e.collectCallIndexKeys(index, idx, child, keySet); err != nil { + if err := e.collectCallIndexKeys(index, idx, isDefaultIndex, child, keySet); err != nil { return errors.Wrapf(err, "translating %s", child) } } - if filter, ok, err := c.CallArg("filter"); ok { - if err != nil { - return errors.Wrap(err, "getting filter call") - } - err = e.collectCallIndexKeys(index, idx, filter, keySet) - if err != nil { - return errors.Wrap(err, "translating filter call") + if callIndex := c.CallIndex(); callIndex == index || (callIndex == "" && isDefaultIndex) { + if filter, ok, err := c.CallArg("filter"); ok { + if err != nil { + return errors.Wrap(err, "getting filter call") + } + err = e.collectCallIndexKeys(index, idx, isDefaultIndex, filter, keySet) + if err != nil { + return errors.Wrap(err, "translating filter call") + } } } return nil } - colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) - if c.Args[colKey] != nil && isString(c.Args[colKey]) { - if value := callArgString(c, colKey); value != "" { - keySet[value] = struct{}{} + if callIndex := c.CallIndex(); callIndex == index || (callIndex == "" && isDefaultIndex) { + colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) + if c.Args[colKey] != nil && isString(c.Args[colKey]) { + if value := callArgString(c, colKey); value != "" { + keySet[value] = struct{}{} + } } } return nil } -func (e *executor) translateCall(index string, idx *Index, c *pql.Call, keyMap map[string]uint64) error { +func (e *executor) translateCall(indexName string, idx *Index, isDefaultIndex bool, c *pql.Call, keyMap map[string]uint64) error { if c.Name == "GroupBy" { - return errors.Wrap(e.translateGroupByCall(index, idx, c, keyMap), "translating GroupBy") + return errors.Wrap(e.translateGroupByCall(indexName, idx, isDefaultIndex, c, keyMap), "translating GroupBy") } // Translate column key. - colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) - if idx.Keys() { - if c.Args[colKey] != nil && !isString(c.Args[colKey]) { - if !isValidID(c.Args[colKey]) { - return errors.Errorf("column value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[colKey]) - } - } else if value := callArgString(c, colKey); value != "" { - c.Args[colKey] = keyMap[value] - } - } else { - if isString(c.Args[colKey]) { - return errors.New("string 'col' value not allowed unless index 'keys' option enabled") - } - } - - // Translate row key, if field is specified & key exists. - if fieldName != "" { - field := idx.Field(fieldName) - if field == nil { - // Instead of returning ErrFieldNotFound here, - // we just return, and don't attempt the translation. - // The assumption is that the non-existent field - // will raise an error downstream when it's used. - return nil - } - - // Bool field keys do not use the translator because there - // are only two possible values. Instead, they are handled - // directly. - if field.Type() == FieldTypeBool { - // TODO: This code block doesn't make sense for a `Rows()` - // queries on a `bool` field. Need to review this better, - // include it in tests, and probably back-port it to Pilosa. - if c.Name != "Rows" { - boolVal, err := callArgBool(c, rowKey) - if err != nil { - return errors.Wrap(err, "getting bool key") + if callIndex := c.CallIndex(); callIndex == indexName || (callIndex == "" && isDefaultIndex) { + colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) + if idx.Keys() { + if c.Args[colKey] != nil && !isString(c.Args[colKey]) { + if !isValidID(c.Args[colKey]) { + return errors.Errorf("column value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[colKey]) } - rowID := falseRowID - if boolVal { - rowID = trueRowID - } - c.Args[rowKey] = rowID - } - } else if field.keys() { - if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { - // allow passing row id directly (this can come in handy, but make sure it is a valid row id) - if !isValidID(c.Args[rowKey]) { - return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) - } - } else if value := callArgString(c, rowKey); value != "" { - id, err := field.TranslateStore().TranslateKey(value) - if err != nil { - return err - } - c.Args[rowKey] = id + } else if value := callArgString(c, colKey); value != "" { + c.Args[colKey] = keyMap[value] } } else { - if isString(c.Args[rowKey]) { - return errors.New("string 'row' value not allowed unless field 'keys' option enabled") + if isString(c.Args[colKey]) { + return errors.New("string 'col' value not allowed unless index 'keys' option enabled") + } + } + + // Translate row key, if field is specified & key exists. + if fieldName != "" { + field := idx.Field(fieldName) + if field == nil { + // Instead of returning ErrFieldNotFound here, + // we just return, and don't attempt the translation. + // The assumption is that the non-existent field + // will raise an error downstream when it's used. + return nil + } + + // Bool field keys do not use the translator because there + // are only two possible values. Instead, they are handled + // directly. + if field.Type() == FieldTypeBool { + // TODO: This code block doesn't make sense for a `Rows()` + // queries on a `bool` field. Need to review this better, + // include it in tests, and probably back-port it to Pilosa. + if c.Name != "Rows" { + boolVal, err := callArgBool(c, rowKey) + if err != nil { + return errors.Wrap(err, "getting bool key") + } + rowID := falseRowID + if boolVal { + rowID = trueRowID + } + c.Args[rowKey] = rowID + } + } else if field.keys() { + if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { + // allow passing row id directly (this can come in handy, but make sure it is a valid row id) + if !isValidID(c.Args[rowKey]) { + return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) + } + } else if value := callArgString(c, rowKey); value != "" { + id, err := field.TranslateStore().TranslateKey(value) + if err != nil { + return err + } + c.Args[rowKey] = id + } + } else { + if isString(c.Args[rowKey]) { + return errors.New("string 'row' value not allowed unless field 'keys' option enabled") + } } } } // Translate child calls. for _, child := range c.Children { - // Possibly change to another index for translation, if this - // call crosses index boundaries. - newIdxName := child.CallIndex() - var newIdx *Index - if newIdxName == "" || newIdxName == index { - newIdxName = index - newIdx = idx - } else { - newIdx = idx.holder.indexes[newIdxName] - if newIdx == nil { - return fmt.Errorf("unknown index %q specified in cross-index call", newIdxName) - } - } - if err := e.translateCall(newIdxName, newIdx, child, keyMap); err != nil { + if err := e.translateCall(indexName, idx, isDefaultIndex, child, keyMap); err != nil { return err } } @@ -3691,34 +3723,22 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call, keyMap m return nil } -func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call, keyMap map[string]uint64) error { +func (e *executor) translateGroupByCall(index string, idx *Index, isDefaultIndex bool, c *pql.Call, keyMap map[string]uint64) error { if c.Name != "GroupBy" { panic("translateGroupByCall called with '" + c.Name + "'") } for _, child := range c.Children { - if err := e.translateCall(index, idx, child, keyMap); err != nil { + if err := e.translateCall(index, idx, isDefaultIndex, child, keyMap); err != nil { return errors.Wrapf(err, "translating %s", child) } } - if filter, ok, err := c.CallArg("filter"); ok { - if err != nil { - return errors.Wrap(err, "getting filter call") - } - err = e.translateCall(index, idx, filter, keyMap) - if err != nil { - return errors.Wrap(err, "translating filter call") - } - } - - if aggregate, ok, err := c.CallArg("aggregate"); ok { - if err != nil { - return errors.Wrap(err, "getting aggregate call") - } - err = e.translateCall(index, idx, aggregate, keyMap) - if err != nil { - return errors.Wrap(err, "translating aggregate call") + for _, arg := range c.Args { + if arg, ok := arg.(*pql.Call); ok { + if err := e.translateCall(index, idx, isDefaultIndex, arg, keyMap); err != nil { + return errors.Wrap(err, "translating group by arg") + } } } diff --git a/executor_internal_test.go b/executor_internal_test.go index b4487dca6..0c1ccd4ba 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -61,7 +61,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, c, make(map[string]uint64)) + err = e.translateGroupByCall("i", idx, true, c, make(map[string]uint64)) if err != nil { t.Fatalf("translating call: %v", err) } @@ -125,7 +125,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, c, make(map[string]uint64)) + err = e.translateGroupByCall("i", idx, true, c, make(map[string]uint64)) if err == nil { t.Fatalf("expected error, but translated call is '%s", c) } diff --git a/index.go b/index.go index a8adfc674..c54aa9814 100644 --- a/index.go +++ b/index.go @@ -164,7 +164,6 @@ func (i *Index) Open() (err error) { return errors.Wrap(err, "opening attrstore") } - // TODO(BBJ): Support non-default partition counts. i.logger.Debugf("open translate store for index: %s", i.name) for partitionID := 0; partitionID < i.partitionN; partitionID++ { store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.partitionN) From 1f6910b0b8958b47c49c763c8638d6dccb3ca636 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 4 Jan 2020 16:04:08 -0700 Subject: [PATCH 15/27] fix group by test --- cluster.go | 1 - executor_internal_test.go | 7 +------ go.mod | 10 +--------- go.sum | 5 +++++ 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/cluster.go b/cluster.go index 746a8e4c9..5763d740c 100644 --- a/cluster.go +++ b/cluster.go @@ -2215,7 +2215,6 @@ func (c *cluster) translateIndexIDSet(ctx context.Context, indexName string, idS return idMap, nil } - // ClusterStatus describes the status of the cluster including its // state and node topology. type ClusterStatus struct { diff --git a/executor_internal_test.go b/executor_internal_test.go index 0c1ccd4ba..c3793f9dd 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -27,12 +27,7 @@ import ( func TestExecutor_TranslateGroupByCall(t *testing.T) { holder := NewHolder(DefaultPartitionN) - cluster := newCluster() - cluster.holder = holder - cluster.Node = &Node{ID: "node1", URI: NewTestURIFromHostPort("node1", 0)} - if err := cluster.addNode(cluster.Node); err != nil { - t.Fatal(err) - } + cluster := NewTestCluster(1) e := &executor{ Holder: holder, diff --git a/go.mod b/go.mod index 9366e18cd..0c4bb020b 100644 --- a/go.mod +++ b/go.mod @@ -8,40 +8,32 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 - github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.3.2 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 + github.com/pilosa/pilosa v1.4.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible - github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 - github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect github.com/youtube/vitess v2.1.1+incompatible // indirect - go.uber.org/atomic v1.4.0 // indirect - golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect - golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 diff --git a/go.sum b/go.sum index 1b1b7af2f..f4afb1a6e 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -103,6 +104,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= +github.com/pilosa/pilosa v1.4.0/go.mod h1:NSTtTprtb5MSgCs4mcNqeQ2JdIMpInOi4DEImxGJeTs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -121,6 +124,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= @@ -153,6 +157,7 @@ github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= From cda272993550a4777a25480e7a2c7359788c20bd Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 6 Jan 2020 13:52:55 -0700 Subject: [PATCH 16/27] fix bitdepth race --- executor.go | 24 ++++++++++-------------- field.go | 21 ++++++++++----------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/executor.go b/executor.go index 125f46358..12f0897fd 100644 --- a/executor.go +++ b/executor.go @@ -3588,13 +3588,11 @@ func (e *executor) collectCallIndexNameMap(ctx context.Context, defaultIndexName m[callIndex] = struct{}{} if c.Name == "GroupBy" { - if filter, ok, err := c.CallArg("filter"); ok { - if err != nil { - return errors.Wrap(err, "getting filter call") - } - err := e.collectCallIndexNameMap(ctx, defaultIndexName, filter, m) - if err != nil { - return errors.Wrap(err, "collecting filter call index name") + for _, arg := range c.Args { + if arg, ok := arg.(*pql.Call); ok { + if err := e.collectCallIndexNameMap(ctx, defaultIndexName, arg, m); err != nil { + return errors.Wrap(err, "collecting group by call index name") + } } } } @@ -3617,13 +3615,11 @@ func (e *executor) collectCallIndexKeys(index string, idx *Index, isDefaultIndex } if callIndex := c.CallIndex(); callIndex == index || (callIndex == "" && isDefaultIndex) { - if filter, ok, err := c.CallArg("filter"); ok { - if err != nil { - return errors.Wrap(err, "getting filter call") - } - err = e.collectCallIndexKeys(index, idx, isDefaultIndex, filter, keySet) - if err != nil { - return errors.Wrap(err, "translating filter call") + for _, arg := range c.Args { + if arg, ok := arg.(*pql.Call); ok { + if err := e.collectCallIndexKeys(index, idx, isDefaultIndex, arg, keySet); err != nil { + return errors.Wrap(err, "translating group by arg call") + } } } } diff --git a/field.go b/field.go index f579bef4a..75691be52 100644 --- a/field.go +++ b/field.go @@ -1490,21 +1490,20 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO requiredDepth = v } // Increase bit depth if required. - f.mu.RLock() - bitDepth := bsig.BitDepth - f.mu.RUnlock() - if requiredDepth > bitDepth { - if err := func() error { - f.mu.Lock() - defer f.mu.Unlock() + if err := func() error { + f.mu.Lock() + defer f.mu.Unlock() + bitDepth := bsig.BitDepth + if requiredDepth > bitDepth { bsig.BitDepth = requiredDepth f.options.BitDepth = requiredDepth return f.saveMeta() - }(); err != nil { - return errors.Wrap(err, "increasing bsi bit depth") + } else { + requiredDepth = bitDepth } - } else { - requiredDepth = bitDepth + return nil + }(); err != nil { + return errors.Wrap(err, "increasing bsi bit depth") } // Import into each fragment. From 54679c12c4364de125b51f0daaab21dd9a5dee31 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 14 Jan 2020 22:10:12 -0600 Subject: [PATCH 17/27] post merge, needs review of TODOs --- cluster.go | 2 +- executor.go | 24 ++++++++++++++++++++++-- executor_test.go | 1 + field.go | 2 +- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 5763d740c..140babc11 100644 --- a/cluster.go +++ b/cluster.go @@ -869,7 +869,7 @@ func shardPartition(index string, shard uint64, partitionN int) int { return int(h.Sum64() % uint64(partitionN)) } -// keyPartition returns the partition that a shard belongs to. +// keyPartition returns the partition that a key belongs to. func (c *cluster) keyPartition(index, key string) int { return keyPartition(index, key, c.partitionN) } diff --git a/executor.go b/executor.go index a36ea9d0e..a832dab5e 100644 --- a/executor.go +++ b/executor.go @@ -3689,7 +3689,27 @@ func (e *executor) translateCall(indexName string, idx *Index, isDefaultIndex bo c.Args[rowKey] = rowID } } else if field.Keys() { - if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { + if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { + // In the case where a field has a foreign index with keys, + // allow `== "key"` or `!= "key"` to be used against the BSI + // field. + cond := c.Args[rowKey].(*pql.Condition) + if isString(cond.Value) { + switch cond.Op { + case pql.EQ, pql.NEQ: + id, err := field.TranslateStore().TranslateKey(cond.Value.(string)) + if err != nil { + return errors.Wrap(err, "translating key") + } + c.Args[rowKey] = &pql.Condition{ + Op: cond.Op, + Value: id, + } + default: + return errors.Errorf("conditional is not supported with string predicates: %s", cond.Op) + } + } + } else if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { // allow passing row id directly (this can come in handy, but make sure it is a valid row id) if !isValidID(c.Args[rowKey]) { return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) @@ -3854,7 +3874,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res // In the case where a field/foreignIndex doesn't exist, // fall back to using the index translateStore. if store == nil && idx.Keys() { - store = idx.translateStore + store = nil // TODO: this may need to be idx.TranslateStore(?) } if store != nil { diff --git a/executor_test.go b/executor_test.go index e3002a70c..b7d3e3d14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3912,6 +3912,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { t.Fatalf("unexpected columns: %v", neq.Columns()) } + // TODO: this test is failing because field `color` is being associated to index `parent` join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row) if !reflect.DeepEqual(join.Keys, []string{"one"}) { t.Fatalf("unexpected keys: %v", join.Keys) diff --git a/field.go b/field.go index 0c41534d0..f47f79342 100644 --- a/field.go +++ b/field.go @@ -565,7 +565,7 @@ func (f *Field) applyForeignIndex() error { return errors.Wrapf(ErrForeignIndexNotFound, "%s", f.options.ForeignIndex) } else if foreignIndex.Keys() { f.usesKeys = true - f.translateStore = foreignIndex.translateStore + f.translateStore = foreignIndex.TranslateStore(0) // TODO: this is wrong return nil } return f.applyTranslateStore() From 4020f8c73e55974244a01ff609ed181733c42aa6 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 15 Jan 2020 14:28:06 -0600 Subject: [PATCH 18/27] fix cross-index translation --- executor.go | 379 +++++++++++++++++--------------------- executor_internal_test.go | 4 +- executor_test.go | 3 +- pql/ast.go | 2 + test/pilosa.go | 9 + 5 files changed, 181 insertions(+), 216 deletions(-) diff --git a/executor.go b/executor.go index a832dab5e..a11ee4cd0 100644 --- a/executor.go +++ b/executor.go @@ -185,7 +185,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // Translate query keys to ids, if necessary. // No need to translate a remote call. if !opt.Remote { - if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil { + if err := e.translateCalls(ctx, index, q.Calls); err != nil { return resp, err } else if err := validateQueryContext(ctx); err != nil { return resp, err @@ -3526,279 +3526,234 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } } -func (e *executor) translateCalls(ctx context.Context, defaultIndexName string, defaultIdx *Index, calls []*pql.Call) (err error) { +func (e *executor) translateCalls(ctx context.Context, defaultIndexName string, calls []*pql.Call) (err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateCalls") defer span.Finish() // Generate a list of all used - indexNameMap := make(map[string]struct{}) + keySets := make(map[string]map[string]struct{}) + keySets[defaultIndexName] = make(map[string]struct{}) for i := range calls { - if err := e.collectCallIndexNameMap(ctx, defaultIndexName, calls[i], indexNameMap); err != nil { + if err := e.collectCallKeySets(ctx, defaultIndexName, calls[i], keySets); err != nil { return err } } // Perform a separate batch translation for each separate index used. - for indexName := range indexNameMap { - // Determine the target index name. - if indexName == "" { - indexName = defaultIndexName - } - isDefaultIndex := indexName == defaultIndexName - - // Determine the target index. - idx := defaultIdx - if !isDefaultIndex { - idx = idx.holder.indexes[indexName] - if idx == nil { - return fmt.Errorf("unknown index %q specified in cross-index call", indexName) - } + keyMaps := make(map[string]map[string]uint64) + for indexName, keySet := range keySets { + idx := e.Holder.indexes[indexName] + if idx == nil { + return fmt.Errorf("canot find index %q", indexName) } - // Collect all index keys & bulk translate them. - keyMap := make(map[string]uint64) - if idx.Keys() { - keySet := make(map[string]struct{}) - for i := range calls { - if err := e.collectCallIndexKeys(indexName, idx, isDefaultIndex, calls[i], keySet); err != nil { - return err - } - } - - if keyMap, err = e.Cluster.translateIndexKeySet(ctx, indexName, keySet); err != nil { - return err - } + if !idx.Keys() || len(keySets) == 0 { + continue } - - // Translate calls. - for i := range calls { - if err := e.translateCall(indexName, idx, isDefaultIndex, calls[i], keyMap); err != nil { - return err - } + if keyMaps[indexName], err = e.Cluster.translateIndexKeySet(ctx, indexName, keySet); err != nil { + return err } } + + // Translate calls. + for i := range calls { + if err := e.translateCall(defaultIndexName, calls[i], keyMaps); err != nil { + return err + } + } + return nil } -func (e *executor) collectCallIndexNameMap(ctx context.Context, defaultIndexName string, c *pql.Call, m map[string]struct{}) error { - callIndex := c.CallIndex() - if callIndex == "" { - callIndex = defaultIndexName +func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c *pql.Call, m map[string]map[string]struct{}) error { + // Specifying an 'index' call overrides indexes on subsequent calls. + if s := c.CallIndex(); s != "" { + indexName = s } - m[callIndex] = struct{}{} - if c.Name == "GroupBy" { - for _, arg := range c.Args { - if arg, ok := arg.(*pql.Call); ok { - if err := e.collectCallIndexNameMap(ctx, defaultIndexName, arg, m); err != nil { - return errors.Wrap(err, "collecting group by call index name") - } + if m[indexName] == nil { + m[indexName] = make(map[string]struct{}) + } + + // Collect key for this call. + colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) + if c.Args[colKey] != nil && isString(c.Args[colKey]) { + if value := callArgString(c, colKey); value != "" { + m[indexName][value] = struct{}{} + } + } + + // Recursively collect argument calls. + for _, arg := range c.Args { + if arg, ok := arg.(*pql.Call); ok { + if err := e.collectCallKeySets(ctx, indexName, arg, m); err != nil { + return errors.Wrap(err, "collecting group by call index name") } } } + // Recursively collect child calls. for _, child := range c.Children { - if err := e.collectCallIndexNameMap(ctx, defaultIndexName, child, m); err != nil { + if err := e.collectCallKeySets(ctx, indexName, child, m); err != nil { return err } } return nil } -func (e *executor) collectCallIndexKeys(index string, idx *Index, isDefaultIndex bool, c *pql.Call, keySet map[string]struct{}) error { - // Handle group by separately. - if c.Name == "GroupBy" { - for _, child := range c.Children { - if err := e.collectCallIndexKeys(index, idx, isDefaultIndex, child, keySet); err != nil { - return errors.Wrapf(err, "translating %s", child) - } - } - - if callIndex := c.CallIndex(); callIndex == index || (callIndex == "" && isDefaultIndex) { - for _, arg := range c.Args { - if arg, ok := arg.(*pql.Call); ok { - if err := e.collectCallIndexKeys(index, idx, isDefaultIndex, arg, keySet); err != nil { - return errors.Wrap(err, "translating group by arg call") - } - } - } - } - return nil - } - - if callIndex := c.CallIndex(); callIndex == index || (callIndex == "" && isDefaultIndex) { - colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) - if c.Args[colKey] != nil && isString(c.Args[colKey]) { - if value := callArgString(c, colKey); value != "" { - keySet[value] = struct{}{} - } - } - } - return nil -} - -func (e *executor) translateCall(indexName string, idx *Index, isDefaultIndex bool, c *pql.Call, keyMap map[string]uint64) error { - if c.Name == "GroupBy" { - return errors.Wrap(e.translateGroupByCall(indexName, idx, isDefaultIndex, c, keyMap), "translating GroupBy") +func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[string]map[string]uint64) error { + // Specifying an 'index' arg applies to all nested calls. + if s := c.CallIndex(); s != "" { + indexName = s } + keyMap := keyMaps[indexName] // Translate column key. - if callIndex := c.CallIndex(); callIndex == indexName || (callIndex == "" && isDefaultIndex) { - colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) - if idx.Keys() { - if c.Args[colKey] != nil && !isString(c.Args[colKey]) { - if !isValidID(c.Args[colKey]) { - return errors.Errorf("column value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[colKey]) - } - } else if value := callArgString(c, colKey); value != "" { - c.Args[colKey] = keyMap[value] - } - } else { - if isString(c.Args[colKey]) { - return errors.New("string 'col' value not allowed unless index 'keys' option enabled") + colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) + idx := e.Holder.indexes[indexName] + if idx.Keys() { + if c.Args[colKey] != nil && !isString(c.Args[colKey]) { + if !isValidID(c.Args[colKey]) { + return errors.Errorf("column value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[colKey]) } + } else if value := callArgString(c, colKey); value != "" { + c.Args[colKey] = keyMap[value] + } + } else { + if isString(c.Args[colKey]) { + return errors.New("string 'col' value not allowed unless index 'keys' option enabled") + } + } + + // Translate row key, if field is specified & key exists. + if fieldName != "" { + field := idx.Field(fieldName) + if field == nil { + // Instead of returning ErrFieldNotFound here, + // we just return, and don't attempt the translation. + // The assumption is that the non-existent field + // will raise an error downstream when it's used. + return nil } - // Translate row key, if field is specified & key exists. - if fieldName != "" { - field := idx.Field(fieldName) - if field == nil { - // Instead of returning ErrFieldNotFound here, - // we just return, and don't attempt the translation. - // The assumption is that the non-existent field - // will raise an error downstream when it's used. - return nil + // Bool field keys do not use the translator because there + // are only two possible values. Instead, they are handled + // directly. + if field.Type() == FieldTypeBool { + // TODO: This code block doesn't make sense for a `Rows()` + // queries on a `bool` field. Need to review this better, + // include it in tests, and probably back-port it to Pilosa. + if c.Name != "Rows" { + boolVal, err := callArgBool(c, rowKey) + if err != nil { + return errors.Wrap(err, "getting bool key") + } + rowID := falseRowID + if boolVal { + rowID = trueRowID + } + c.Args[rowKey] = rowID } - - // Bool field keys do not use the translator because there - // are only two possible values. Instead, they are handled - // directly. - if field.Type() == FieldTypeBool { - // TODO: This code block doesn't make sense for a `Rows()` - // queries on a `bool` field. Need to review this better, - // include it in tests, and probably back-port it to Pilosa. - if c.Name != "Rows" { - boolVal, err := callArgBool(c, rowKey) - if err != nil { - return errors.Wrap(err, "getting bool key") - } - rowID := falseRowID - if boolVal { - rowID = trueRowID - } - c.Args[rowKey] = rowID - } - } else if field.Keys() { - if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { - // In the case where a field has a foreign index with keys, - // allow `== "key"` or `!= "key"` to be used against the BSI - // field. - cond := c.Args[rowKey].(*pql.Condition) - if isString(cond.Value) { - switch cond.Op { - case pql.EQ, pql.NEQ: - id, err := field.TranslateStore().TranslateKey(cond.Value.(string)) - if err != nil { - return errors.Wrap(err, "translating key") - } - c.Args[rowKey] = &pql.Condition{ - Op: cond.Op, - Value: id, - } - default: - return errors.Errorf("conditional is not supported with string predicates: %s", cond.Op) + } else if field.Keys() { + if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { + // In the case where a field has a foreign index with keys, + // allow `== "key"` or `!= "key"` to be used against the BSI + // field. + cond := c.Args[rowKey].(*pql.Condition) + if isString(cond.Value) { + switch cond.Op { + case pql.EQ, pql.NEQ: + id, err := field.TranslateStore().TranslateKey(cond.Value.(string)) + if err != nil { + return errors.Wrap(err, "translating key") } + c.Args[rowKey] = &pql.Condition{ + Op: cond.Op, + Value: id, + } + default: + return errors.Errorf("conditional is not supported with string predicates: %s", cond.Op) } - } else if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { - // allow passing row id directly (this can come in handy, but make sure it is a valid row id) - if !isValidID(c.Args[rowKey]) { - return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) - } - } else if value := callArgString(c, rowKey); value != "" { - id, err := field.TranslateStore().TranslateKey(value) - if err != nil { - return err - } - c.Args[rowKey] = id } - } else { - if isString(c.Args[rowKey]) { - return errors.New("string 'row' value not allowed unless field 'keys' option enabled") + } else if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { + // allow passing row id directly (this can come in handy, but make sure it is a valid row id) + if !isValidID(c.Args[rowKey]) { + return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) } + } else if value := callArgString(c, rowKey); value != "" { + id, err := field.TranslateStore().TranslateKey(value) + if err != nil { + return err + } + c.Args[rowKey] = id + } + } else { + if isString(c.Args[rowKey]) { + return errors.New("string 'row' value not allowed unless field 'keys' option enabled") } } } // Translate child calls. for _, child := range c.Children { - if err := e.translateCall(indexName, idx, isDefaultIndex, child, keyMap); err != nil { + if err := e.translateCall(indexName, child, keyMaps); err != nil { return err } } - return nil -} - -func (e *executor) translateGroupByCall(index string, idx *Index, isDefaultIndex bool, c *pql.Call, keyMap map[string]uint64) error { - if c.Name != "GroupBy" { - panic("translateGroupByCall called with '" + c.Name + "'") - } - - for _, child := range c.Children { - if err := e.translateCall(index, idx, isDefaultIndex, child, keyMap); err != nil { - return errors.Wrapf(err, "translating %s", child) - } - } - + // Translate call args. for _, arg := range c.Args { if arg, ok := arg.(*pql.Call); ok { - if err := e.translateCall(index, idx, isDefaultIndex, arg, keyMap); err != nil { - return errors.Wrap(err, "translating group by arg") + if err := e.translateCall(indexName, arg, keyMaps); err != nil { + return errors.Wrap(err, "translating arg") } } } - prev, ok := c.Args["previous"] - if !ok { - return nil // nothing else to be translated - } - previous, ok := prev.([]interface{}) - if !ok { - return errors.Errorf("'previous' argument must be list, but got %T", prev) - } - if len(c.Children) != len(previous) { - return errors.Errorf("mismatched lengths for previous: %d and children: %d in %s", len(previous), len(c.Children), c) - } - - fields := make([]*Field, len(c.Children)) - for i, child := range c.Children { - fieldname := callArgString(child, "_field") - field := idx.Field(fieldname) - if field == nil { - return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) + // GroupBy-specific call translation. + if c.Name == "GroupBy" { + prev, ok := c.Args["previous"] + if !ok { + return nil // nothing else to be translated } - fields[i] = field - } - - for i, field := range fields { - prev := previous[i] - if field.Keys() { - prevStr, ok := prev.(string) - if !ok { - return errors.New("prev value must be a string when field 'keys' option enabled") - } - id, err := field.TranslateStore().TranslateKey(prevStr) - if err != nil { - return errors.Wrapf(err, "translating row key '%s'", prevStr) - } - previous[i] = id - } else { - if prevStr, ok := prev.(string); ok { - return errors.Errorf("got string row val '%s' in 'previous' for field %s which doesn't use string keys", prevStr, field.Name()) - } + previous, ok := prev.([]interface{}) + if !ok { + return errors.Errorf("'previous' argument must be list, but got %T", prev) + } + if len(c.Children) != len(previous) { + return errors.Errorf("mismatched lengths for previous: %d and children: %d in %s", len(previous), len(c.Children), c) } + fields := make([]*Field, len(c.Children)) + for i, child := range c.Children { + fieldname := callArgString(child, "_field") + field := idx.Field(fieldname) + if field == nil { + return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) + } + fields[i] = field + } + + for i, field := range fields { + prev := previous[i] + if field.Keys() { + prevStr, ok := prev.(string) + if !ok { + return errors.New("prev value must be a string when field 'keys' option enabled") + } + id, err := field.TranslateStore().TranslateKey(prevStr) + if err != nil { + return errors.Wrapf(err, "translating row key '%s'", prevStr) + } + previous[i] = id + } else { + if prevStr, ok := prev.(string); ok { + return errors.Errorf("got string row val '%s' in 'previous' for field %s which doesn't use string keys", prevStr, field.Name()) + } + } + } } + return nil } diff --git a/executor_internal_test.go b/executor_internal_test.go index c3793f9dd..8f2a5eafd 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -56,7 +56,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, true, c, make(map[string]uint64)) + err = e.translateCall("i", c, make(map[string]map[string]uint64)) if err != nil { t.Fatalf("translating call: %v", err) } @@ -120,7 +120,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("parsing query: %v", err) } c := query.Calls[0] - err = e.translateGroupByCall("i", idx, true, c, make(map[string]uint64)) + err = e.translateCall("i", c, make(map[string]map[string]uint64)) if err == nil { t.Fatalf("expected error, but translated call is '%s", c) } diff --git a/executor_test.go b/executor_test.go index b7d3e3d14..afd322eb6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3712,7 +3712,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { Query: tst.query, }) if err != nil { - t.Fatalf("got an error %v", err) + t.Fatal(err) } results := r.Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, tst.expected, results) @@ -3912,7 +3912,6 @@ func TestExecutor_ForeignIndex(t *testing.T) { t.Fatalf("unexpected columns: %v", neq.Columns()) } - // TODO: this test is failing because field `color` is being associated to index `parent` join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row) if !reflect.DeepEqual(join.Keys, []string{"one"}) { t.Fatalf("unexpected keys: %v", join.Keys) diff --git a/pql/ast.go b/pql/ast.go index 9d23c12b6..9855f5b96 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -338,6 +338,8 @@ var callInfoByFunc = map[string]callInfo{ "Row": {allowUnknown: true}, "Range": {allowUnknown: true}, + "Distinct": {allowUnknown: true}, + // allow only "field=X" cases with string field names "Max": allowField, "Min": allowField, diff --git a/test/pilosa.go b/test/pilosa.go index 73935290c..34880e627 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -137,6 +137,7 @@ func (m *Command) Reopen() error { // MustCreateIndex uses this command's API to create an index and fails the test // if there is an error. func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexOptions) *pilosa.Index { + tb.Helper() idx, err := m.API.CreateIndex(context.Background(), name, opts) if err != nil { tb.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err) @@ -147,6 +148,7 @@ func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexO // MustCreateField uses this command's API to create the field. The index must // already exist - it fails the test if there is an error. func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pilosa.FieldOption) *pilosa.Field { + tb.Helper() f, err := m.API.CreateField(context.Background(), index, field, opts...) if err != nil { tb.Fatalf("creating field: %s in index: %s err: %v", field, index, err) @@ -157,6 +159,7 @@ func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pi // MustQuery uses this command's API to execute the given query request, failing // if Query returns a non-nil error, otherwise returning the QueryResponse. func (m *Command) MustQuery(tb testing.TB, req *pilosa.QueryRequest) pilosa.QueryResponse { + tb.Helper() resp, err := m.API.Query(context.Background(), req) if err != nil { tb.Fatalf("making query: %v, err: %v", req, err) @@ -248,6 +251,7 @@ type Cluster []*Command // Query executes an API.Query through one of the cluster's node's API. It fails // the test if there is an error. func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse { + t.Helper() if len(c) == 0 { t.Fatal("must have at least one node in cluster to query") } @@ -256,6 +260,7 @@ func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse { } func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) { + t.Helper() byShard := make(map[uint64][][2]uint64) for _, rowcol := range rowcols { shard := rowcol[1] / pilosa.ShardWidth @@ -296,6 +301,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint // CreateField creates the index (if necessary) and field specified. func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { + t.Helper() idx, err := c[0].API.CreateIndex(context.Background(), index, iopts) if err != nil && !strings.Contains(err.Error(), "index already exists") { t.Fatalf("creating index: %v", err) @@ -344,6 +350,7 @@ func (c Cluster) Close() error { // MustNewCluster creates a new cluster func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { + tb.Helper() c, err := newCluster(size, opts...) if err != nil { tb.Fatalf("new cluster: %v", err) @@ -391,6 +398,7 @@ func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { // MustRunCluster creates and starts a new cluster func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { + tb.Helper() c, err := runCluster(size, opts...) if err != nil { tb.Fatalf("run cluster: %v", err) @@ -429,6 +437,7 @@ func MustDo(method, urlStr string, body string) *httpResponse { } func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { + t.Helper() if len(results) != len(expected) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } From 10ccb6d5238ca29962a54e93b4a09f1186446489 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 16 Jan 2020 13:53:12 -0600 Subject: [PATCH 19/27] translate foreign index --- executor.go | 58 ++++++++++++++++++++++++++++++++++++++++++++--------- field.go | 27 +++++++++---------------- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/executor.go b/executor.go index a11ee4cd0..7b79fd11c 100644 --- a/executor.go +++ b/executor.go @@ -3576,13 +3576,33 @@ func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c * } // Collect key for this call. - colKey, _, _ := c.TranslateInfo(columnLabel, rowLabel) + colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) if c.Args[colKey] != nil && isString(c.Args[colKey]) { if value := callArgString(c, colKey); value != "" { m[indexName][value] = struct{}{} } } + // Collection foreign index keys. + if fieldName != "" { + idx := e.Holder.indexes[indexName] + if field := idx.Field(fieldName); field != nil && field.ForeignIndex() != "" { + foreignIndexName := field.ForeignIndex() + if m[foreignIndexName] == nil { + m[foreignIndexName] = make(map[string]struct{}) + } + + if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { + cond := c.Args[rowKey].(*pql.Condition) + if isString(cond.Value) { + m[foreignIndexName][cond.Value.(string)] = struct{}{} + } + } else if value := callArgString(c, rowKey); value != "" { + m[foreignIndexName][value] = struct{}{} + } + } + } + // Recursively collect argument calls. for _, arg := range c.Args { if arg, ok := arg.(*pql.Call); ok { @@ -3601,7 +3621,7 @@ func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c * return nil } -func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[string]map[string]uint64) error { +func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[string]map[string]uint64) (err error) { // Specifying an 'index' arg applies to all nested calls. if s := c.CallIndex(); s != "" { indexName = s @@ -3636,6 +3656,17 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri return nil } + // Determine if foreign index is being used for translation. + useKeys := field.Keys() + foreignIndexName := field.ForeignIndex() + if foreignIndexName != "" { + foreignIndex := e.Holder.indexes[foreignIndexName] + if foreignIndex == nil { + return errors.Errorf("foreign index not found: %q", foreignIndexName) + } + useKeys = foreignIndex.Keys() + } + // Bool field keys do not use the translator because there // are only two possible values. Instead, they are handled // directly. @@ -3654,7 +3685,7 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri } c.Args[rowKey] = rowID } - } else if field.Keys() { + } else if useKeys { if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { // In the case where a field has a foreign index with keys, // allow `== "key"` or `!= "key"` to be used against the BSI @@ -3663,10 +3694,15 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri if isString(cond.Value) { switch cond.Op { case pql.EQ, pql.NEQ: - id, err := field.TranslateStore().TranslateKey(cond.Value.(string)) - if err != nil { - return errors.Wrap(err, "translating key") + var id uint64 + if foreignIndexName != "" { + id = keyMaps[foreignIndexName][cond.Value.(string)] + } else { + if id, err = field.TranslateStore().TranslateKey(cond.Value.(string)); err != nil { + return errors.Wrap(err, "translating key") + } } + c.Args[rowKey] = &pql.Condition{ Op: cond.Op, Value: id, @@ -3681,9 +3717,13 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri return errors.Errorf("row value must be a string or non-negative integer, but got: %v of %[1]T", c.Args[rowKey]) } } else if value := callArgString(c, rowKey); value != "" { - id, err := field.TranslateStore().TranslateKey(value) - if err != nil { - return err + var id uint64 + if foreignIndexName != "" { + id = keyMaps[foreignIndexName][value] + } else { + if id, err = field.TranslateStore().TranslateKey(value); err != nil { + return err + } } c.Args[rowKey] = id } diff --git a/field.go b/field.go index f47f79342..149ae65c3 100644 --- a/field.go +++ b/field.go @@ -522,14 +522,8 @@ func (f *Field) Open() error { // If the field has a foreign index, and that index uses keys, // then use that index's translateStore instead. - if f.options.ForeignIndex != "" { - if err := f.holder.checkForeignIndex(f); err != nil { - return errors.Wrap(err, "checking foreign index") - } - } else { - if err := f.applyTranslateStore(); err != nil { - return errors.Wrap(err, "applying translate store") - } + if err := f.applyTranslateStore(); err != nil { + return errors.Wrap(err, "applying translate store") } return nil @@ -554,21 +548,18 @@ func (f *Field) applyTranslateStore() error { return nil } -// applyForeignIndex sets the field's translateStore -// to that of a foreign index in the case where the -// foreign index uses keys. If the foreign index does -// not use keys, it falls back to applying the field's -// default translate store. func (f *Field) applyForeignIndex() error { foreignIndex := f.holder.Index(f.options.ForeignIndex) if foreignIndex == nil { return errors.Wrapf(ErrForeignIndexNotFound, "%s", f.options.ForeignIndex) - } else if foreignIndex.Keys() { - f.usesKeys = true - f.translateStore = foreignIndex.TranslateStore(0) // TODO: this is wrong - return nil } - return f.applyTranslateStore() + return nil +} + +// ForeignIndex returns the foreign index name attached to the field. +// Returns blank string if no foreign index exists. +func (f *Field) ForeignIndex() string { + return f.options.ForeignIndex } var fieldQueue = make(chan struct{}, 16) From 0ba5b48fca0c07bafab572f7ae370bd6add37ae9 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 16 Jan 2020 22:23:29 -0600 Subject: [PATCH 20/27] Field.ForeignIndex translation on ImportValue() --- api.go | 36 +++++++++++++++++++++++++++--------- executor.go | 2 +- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/api.go b/api.go index 2a74f3b11..666bb4692 100644 --- a/api.go +++ b/api.go @@ -1086,18 +1086,36 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . req.Shard = math.MaxUint64 } + // Determine if foreign index is being used for translation. + useKeys := field.Keys() + foreignIndexName := field.ForeignIndex() + if foreignIndexName != "" { + foreignIndex := api.holder.indexes[foreignIndexName] + if foreignIndex == nil { + return errors.Errorf("foreign index not found: %q", foreignIndexName) + } + useKeys = foreignIndex.Keys() + } + // Translate values when the field uses keys (for example, when // the field has a ForeignIndex with keys). - if field.Keys() { - uints, err := field.translateStore.TranslateKeys(req.StringValues) - if err != nil { - return errors.Wrap(err, "translating string values") + if useKeys { + keySet := make(map[string]struct{}) + for i := range req.StringValues { + keySet[req.StringValues[i]] = struct{}{} } - // Because the BSI field supports negative value, we have to - // convert the slice of uint64 keys to a slice of int64. - ints := make([]int64, len(uints)) - for i := range uints { - ints[i] = int64(uints[i]) + + // Perform a separate batch translation for each separate index used. + keyMap := make(map[string]uint64) + if keyMap, err = api.cluster.translateIndexKeySet(ctx, foreignIndexName, keySet); err != nil { + return err + } + + // Because the BSI field supports negative values, we have to + // convert the uint64 keys to a slice of int64. + ints := make([]int64, len(req.StringValues)) + for i := range req.StringValues { + ints[i] = int64(keyMap[req.StringValues[i]]) } req.Values = ints } diff --git a/executor.go b/executor.go index 7b79fd11c..b6120662e 100644 --- a/executor.go +++ b/executor.go @@ -3544,7 +3544,7 @@ func (e *executor) translateCalls(ctx context.Context, defaultIndexName string, for indexName, keySet := range keySets { idx := e.Holder.indexes[indexName] if idx == nil { - return fmt.Errorf("canot find index %q", indexName) + return fmt.Errorf("cannot find index %q", indexName) } if !idx.Keys() || len(keySets) == 0 { From 90a2e116a7cdfc9073eec7ede0467a60b7172d7c Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Jan 2020 11:10:42 -0600 Subject: [PATCH 21/27] update translateResult to translate foreign index keys on SignedRow results --- executor.go | 57 +++++++++++++++++++++++++++++------------------- executor_test.go | 30 ++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/executor.go b/executor.go index b6120662e..e3a064754 100644 --- a/executor.go +++ b/executor.go @@ -3857,34 +3857,47 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res // make the return type for an int field with a ForeignIndex be // a *Row instead (because it should always be positive). case SignedRow: - var store TranslateStore + sr, err := func() (*SignedRow, error) { + fieldName := callArgString(call, "field") + if fieldName == "" { + return nil, nil + } - if fieldName := callArgString(call, "field"); fieldName != "" { field := idx.Field(fieldName) - if field != nil && field.Keys() { - store = field.TranslateStore() + if field == nil { + return nil, nil } - } - // In the case where a field/foreignIndex doesn't exist, - // fall back to using the index translateStore. - if store == nil && idx.Keys() { - store = nil // TODO: this may need to be idx.TranslateStore(?) - } - - if store != nil { - rslt := result.Pos - other := &Row{Attrs: rslt.Attrs} - for _, segment := range rslt.Segments() { - for _, col := range segment.Columns() { - key, err := store.TranslateID(col) - if err != nil { - return nil, err - } - other.Keys = append(other.Keys, key) + // Determine if foreign index is being used for translation. + useKeys := field.Keys() + foreignIndexName := field.ForeignIndex() + if foreignIndexName != "" { + foreignIndex := e.Holder.indexes[foreignIndexName] + if foreignIndex == nil { + return nil, errors.Errorf("foreign index not found: %q", foreignIndexName) } + useKeys = foreignIndex.Keys() } - return SignedRow{Pos: other}, nil + + if useKeys { + rslt := result.Pos + other := &Row{Attrs: rslt.Attrs} + for _, segment := range rslt.Segments() { + keys, err := e.Cluster.translateIndexIDs(context.Background(), foreignIndexName, segment.Columns()) + if err != nil { + return nil, errors.Wrap(err, "translating index ids") + } + other.Keys = append(other.Keys, keys...) + } + return &SignedRow{Pos: other}, nil + } + + return nil, nil + }() + if err != nil { + return nil, err + } else if sr != nil { + return *sr, nil } case PairField: diff --git a/executor_test.go b/executor_test.go index afd322eb6..0ec94efe4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3898,7 +3898,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { `) distinct := c.Query(t, "child", `Distinct(index="child", field="parent_id")`).Results[0].(pilosa.SignedRow) - if !reflect.DeepEqual(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { + if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { t.Fatalf("unexpected keys: %v", distinct.Pos.Keys) } @@ -3918,6 +3918,34 @@ func TestExecutor_ForeignIndex(t *testing.T) { } } +// sameStringSlice is a helper function which compares two string +// slices without enforcing order. +func sameStringSlice(x, y []string) bool { + if len(x) != len(y) { + return false + } + // create a map of string -> int + diff := make(map[string]int, len(x)) + for _, _x := range x { + // 0 value for int is 0, so just increment a counter for the string + diff[_x]++ + } + for _, _y := range y { + // If the string _y is not in diff bail out early + if _, ok := diff[_y]; !ok { + return false + } + diff[_y] -= 1 + if diff[_y] == 0 { + delete(diff, _y) + } + } + if len(diff) == 0 { + return true + } + return false +} + func TestExecutor_Execute_GroupBy(t *testing.T) { groupByTest := func(t *testing.T, clusterSize int) { c := test.MustRunCluster(t, 1) From efffa39c2c0c751aa6eccff0ea02ea50fa766a3d Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Jan 2020 11:42:21 -0600 Subject: [PATCH 22/27] check foreign index on field open --- field.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/field.go b/field.go index 149ae65c3..86cf50b64 100644 --- a/field.go +++ b/field.go @@ -526,6 +526,14 @@ func (f *Field) Open() error { return errors.Wrap(err, "applying translate store") } + // If the field has a foreign index, make sure the index + // exists. + if f.options.ForeignIndex != "" { + if err := f.holder.checkForeignIndex(f); err != nil { + return errors.Wrap(err, "checking foreign index") + } + } + return nil }(); err != nil { f.Close() @@ -548,6 +556,14 @@ func (f *Field) applyTranslateStore() error { return nil } +// applyForeignIndex used to set the field's translateStore to +// that of the foreign index, but since moving to partitioned +// translate stores on indexes, that doesn't happen anymore. +// So now all this method does is check that the foreign index +// actually exists. If we decided this was unnecessary (which +// it kind of is), we could remove the field.holder and all +// the logic which does this check on holder open after all +// indexes have opened. func (f *Field) applyForeignIndex() error { foreignIndex := f.holder.Index(f.options.ForeignIndex) if foreignIndex == nil { From f001ad199f60fff349b80be2f5da2b472f763729 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Jan 2020 11:58:15 -0600 Subject: [PATCH 23/27] use translateIndexKeys instead of translateIndexKeySet in ImportValue() --- api.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index 666bb4692..2049d2abe 100644 --- a/api.go +++ b/api.go @@ -1100,22 +1100,17 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // Translate values when the field uses keys (for example, when // the field has a ForeignIndex with keys). if useKeys { - keySet := make(map[string]struct{}) - for i := range req.StringValues { - keySet[req.StringValues[i]] = struct{}{} - } - - // Perform a separate batch translation for each separate index used. - keyMap := make(map[string]uint64) - if keyMap, err = api.cluster.translateIndexKeySet(ctx, foreignIndexName, keySet); err != nil { + // Perform translation. + uints, err := api.cluster.translateIndexKeys(ctx, foreignIndexName, req.StringValues) + if err != nil { return err } // Because the BSI field supports negative values, we have to // convert the uint64 keys to a slice of int64. - ints := make([]int64, len(req.StringValues)) - for i := range req.StringValues { - ints[i] = int64(keyMap[req.StringValues[i]]) + ints := make([]int64, len(uints)) + for i := range uints { + ints[i] = int64(uints[i]) } req.Values = ints } From b620e37e51820c2345b0796dc92fd9368dc795bd Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Jan 2020 14:41:41 -0600 Subject: [PATCH 24/27] move the foreign index key check into applyTranslateStore() --- api.go | 15 ++------------- executor.go | 29 ++++------------------------- field.go | 11 +++++++++-- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/api.go b/api.go index 2049d2abe..ee24823a1 100644 --- a/api.go +++ b/api.go @@ -1086,22 +1086,11 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . req.Shard = math.MaxUint64 } - // Determine if foreign index is being used for translation. - useKeys := field.Keys() - foreignIndexName := field.ForeignIndex() - if foreignIndexName != "" { - foreignIndex := api.holder.indexes[foreignIndexName] - if foreignIndex == nil { - return errors.Errorf("foreign index not found: %q", foreignIndexName) - } - useKeys = foreignIndex.Keys() - } - // Translate values when the field uses keys (for example, when // the field has a ForeignIndex with keys). - if useKeys { + if field.Keys() { // Perform translation. - uints, err := api.cluster.translateIndexKeys(ctx, foreignIndexName, req.StringValues) + uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues) if err != nil { return err } diff --git a/executor.go b/executor.go index e3a064754..d2c23da46 100644 --- a/executor.go +++ b/executor.go @@ -3656,17 +3656,6 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri return nil } - // Determine if foreign index is being used for translation. - useKeys := field.Keys() - foreignIndexName := field.ForeignIndex() - if foreignIndexName != "" { - foreignIndex := e.Holder.indexes[foreignIndexName] - if foreignIndex == nil { - return errors.Errorf("foreign index not found: %q", foreignIndexName) - } - useKeys = foreignIndex.Keys() - } - // Bool field keys do not use the translator because there // are only two possible values. Instead, they are handled // directly. @@ -3685,7 +3674,8 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri } c.Args[rowKey] = rowID } - } else if useKeys { + } else if field.Keys() { + foreignIndexName := field.ForeignIndex() if c.Args[rowKey] != nil && isCondition(c.Args[rowKey]) { // In the case where a field has a foreign index with keys, // allow `== "key"` or `!= "key"` to be used against the BSI @@ -3868,22 +3858,11 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, nil } - // Determine if foreign index is being used for translation. - useKeys := field.Keys() - foreignIndexName := field.ForeignIndex() - if foreignIndexName != "" { - foreignIndex := e.Holder.indexes[foreignIndexName] - if foreignIndex == nil { - return nil, errors.Errorf("foreign index not found: %q", foreignIndexName) - } - useKeys = foreignIndex.Keys() - } - - if useKeys { + if field.Keys() { rslt := result.Pos other := &Row{Attrs: rslt.Attrs} for _, segment := range rslt.Segments() { - keys, err := e.Cluster.translateIndexIDs(context.Background(), foreignIndexName, segment.Columns()) + keys, err := e.Cluster.translateIndexIDs(context.Background(), field.ForeignIndex(), segment.Columns()) if err != nil { return nil, errors.Wrap(err, "translating index ids") } diff --git a/field.go b/field.go index 86cf50b64..32dc18c22 100644 --- a/field.go +++ b/field.go @@ -520,8 +520,7 @@ func (f *Field) Open() error { return errors.Wrap(err, "opening attrstore") } - // If the field has a foreign index, and that index uses keys, - // then use that index's translateStore instead. + // Apply the field-specific translateStore. if err := f.applyTranslateStore(); err != nil { return errors.Wrap(err, "applying translate store") } @@ -553,6 +552,14 @@ func (f *Field) applyTranslateStore() error { return errors.Wrap(err, "opening field translate store") } f.usesKeys = f.options.Keys + + // In the case where the field has a foreign index, set + // the usesKeys value accordingly. + if foreignIndexName := f.ForeignIndex(); foreignIndexName != "" { + if foreignIndex := f.holder.indexes[foreignIndexName]; foreignIndex != nil { + f.usesKeys = foreignIndex.Keys() + } + } return nil } From d41ee990520f6b108aeb828a3ae51a68764947a1 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Jan 2020 14:46:43 -0600 Subject: [PATCH 25/27] linter fix --- executor_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/executor_test.go b/executor_test.go index 0ec94efe4..7586da42f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3940,10 +3940,7 @@ func sameStringSlice(x, y []string) bool { delete(diff, _y) } } - if len(diff) == 0 { - return true - } - return false + return len(diff) == 0 } func TestExecutor_Execute_GroupBy(t *testing.T) { From e40400b130f506de00b7ad27ccb3802320ecc61c Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 28 Jan 2020 22:41:02 -0600 Subject: [PATCH 26/27] Ensure ForeignIndex key translation happens in API. For Fields with ForeignIndex (which have keys), the API was missing the logic to do that translation against the translateStore of the foreign index. This commit adds that logic, as well as some missing translateStore-related logic in the gRPC code. --- api.go | 15 +++++++++++++ executor.go | 2 ++ field.go | 1 + server/grpc.go | 58 +++++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/api.go b/api.go index ee24823a1..1ffbc7df6 100644 --- a/api.go +++ b/api.go @@ -545,6 +545,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin var err error if field.Keys() { + // TODO: handle case: field.ForeignIndex if rowStr, err = field.TranslateStore().TranslateID(rowID); err != nil { return errors.Wrap(err, "translating row") } @@ -1451,6 +1452,10 @@ func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key str return api.cluster.translateIndexKey(ctx, indexName, key) } +func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error) { + return api.cluster.translateIndexIDs(ctx, indexName, ids) +} + // TranslateKeys handles a TranslateKeyRequest. func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) { var req TranslateKeysRequest @@ -1469,6 +1474,11 @@ func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err e } else { if field := api.holder.Field(req.Index, req.Field); field == nil { return nil, ErrFieldNotFound + } else if fi := field.ForeignIndex(); fi != "" { + ids, err = api.cluster.translateIndexKeys(ctx, fi, req.Keys) + if err != nil { + return nil, err + } } else if ids, err = field.TranslateStore().TranslateKeys(req.Keys); err != nil { return nil, err } @@ -1500,6 +1510,11 @@ func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err er } else { if field := api.holder.Field(req.Index, req.Field); field == nil { return nil, ErrFieldNotFound + } else if fi := field.ForeignIndex(); fi != "" { + keys, err = api.cluster.translateIndexIDs(ctx, fi, req.IDs) + if err != nil { + return nil, err + } } else if keys, err = field.TranslateStore().TranslateIDs(req.IDs); err != nil { return nil, err } diff --git a/executor.go b/executor.go index d2c23da46..c078d4e29 100644 --- a/executor.go +++ b/executor.go @@ -3771,6 +3771,7 @@ func (e *executor) translateCall(indexName string, c *pql.Call, keyMaps map[stri if !ok { return errors.New("prev value must be a string when field 'keys' option enabled") } + // TODO: does this need to take field.ForeignIndex() into consideration? id, err := field.TranslateStore().TranslateKey(prevStr) if err != nil { return errors.Wrapf(err, "translating row key '%s'", prevStr) @@ -3937,6 +3938,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return nil, ErrFieldNotFound } if field.Keys() { + // TODO: does this need to take field.ForeignIndex() into consideration? key, err := field.TranslateStore().TranslateID(g.RowID) if err != nil { return nil, errors.Wrap(err, "translating row ID in Group") diff --git a/field.go b/field.go index 32dc18c22..618a3815d 100644 --- a/field.go +++ b/field.go @@ -576,6 +576,7 @@ func (f *Field) applyForeignIndex() error { if foreignIndex == nil { return errors.Wrapf(ErrForeignIndexNotFound, "%s", f.options.ForeignIndex) } + f.usesKeys = foreignIndex.Keys() return nil } diff --git a/server/grpc.go b/server/grpc.go index 4d410546f..95824c607 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -256,10 +256,31 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer case "int": if field.Keys() { - value, exists, err := field.StringValue(col) - if err != nil { - return errors.Wrap(err, "getting string field value for column") - } else if exists { + var value string + var exists bool + var err error + if fi := field.ForeignIndex(); fi != "" { + // Get the value from the int field. + intVal, ok, err := field.Value(col) + if err != nil { + return errors.Wrap(err, "getting int value") + } else if ok { + vals, err := h.api.TranslateIndexIDs(context.Background(), fi, []uint64{uint64(intVal)}) + if err != nil { + return errors.Wrap(err, "getting keys for ids") + } + if len(vals) > 0 && vals[0] != "" { + value = vals[0] + exists = true + } + } + } else { + value, exists, err = field.StringValue(col) + if err != nil { + return errors.Wrap(err, "getting string field value for column") + } + } + if exists { rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}}) } else { @@ -460,10 +481,31 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } if field.Keys() { - value, exists, err := field.StringValue(id) - if err != nil { - return errors.Wrap(err, "getting string field value for column") - } else if exists { + var value string + var exists bool + var err error + if fi := field.ForeignIndex(); fi != "" { + // Get the value from the int field. + intVal, ok, err := field.Value(id) + if err != nil { + return errors.Wrap(err, "getting int value") + } else if ok { + vals, err := h.api.TranslateIndexIDs(context.Background(), fi, []uint64{uint64(intVal)}) + if err != nil { + return errors.Wrap(err, "getting keys for ids") + } + if len(vals) > 0 && vals[0] != "" { + value = vals[0] + exists = true + } + } + } else { + value, exists, err = field.StringValue(id) + if err != nil { + return errors.Wrap(err, "getting string field value for column") + } + } + if exists { rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: value}}) } else { From 61e527251ab689c99bc1174874389e66fdc04771 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 30 Jan 2020 10:56:03 -0600 Subject: [PATCH 27/27] fix some comments --- executor.go | 2 +- index.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index c078d4e29..473df14b5 100644 --- a/executor.go +++ b/executor.go @@ -3583,7 +3583,7 @@ func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c * } } - // Collection foreign index keys. + // Collect foreign index keys. if fieldName != "" { idx := e.Holder.indexes[indexName] if field := idx.Field(fieldName); field != nil && field.ForeignIndex() != "" { diff --git a/index.go b/index.go index 56cbc0de2..bc6293b1b 100644 --- a/index.go +++ b/index.go @@ -62,8 +62,7 @@ type Index struct { logger logger.Logger snapshotQueue snapshotQueue - // Used for notifying holder when a field is added. - // Also passed to field for foreign-index lookup. + // Passed to field for foreign-index lookup. holder *Holder // Per-partition translation stores