From 8fe1b9a37ca46e9d163259a9b4810975e99ba251 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 8 Feb 2021 23:19:26 -0600 Subject: [PATCH 1/4] store views in etcd via Schemator --- api.go | 2 +- cluster.go | 3 -- disco/disco.go | 25 +++++++++++++++- etcd/embed.go | 75 ++++++++++++++++++++++++++++++++++++++++++++--- field.go | 64 +++++++++++++++++++++++++++++++++------- holder.go | 79 +++++++++++++++++++++++++++++++++++++------------- index.go | 11 +++++-- pilosa.go | 2 ++ server.go | 20 ++++++------- 9 files changed, 227 insertions(+), 54 deletions(-) diff --git a/api.go b/api.go index 4906b80b8..c5a815c8b 100644 --- a/api.go +++ b/api.go @@ -1328,7 +1328,7 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts . return nil } -// Import bulk imports data into a particular index,field,shard. +// ImportWithTx bulk imports data into a particular index,field,shard. func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) error { span, _ := tracing.StartSpanFromContext(ctx, "API.Import") defer span.Finish() diff --git a/cluster.go b/cluster.go index b4c891aa5..222e5c4c4 100644 --- a/cluster.go +++ b/cluster.go @@ -636,9 +636,6 @@ func (c *cluster) remoteSchema() (*Schema, error) { continue } - // TODO: replace following line by: - // ii, err := c.InternalClient.SchemaNode(context.Background(), &n.URI, true) - // after we ii, err := c.InternalClient.SchemaNode(context.Background(), &n.URI, true) if err != nil { return nil, errors.Wrapf(err, "getting schema from %s (%v)", n.ID, n.URI) diff --git a/disco/disco.go b/disco/disco.go index de77f88dd..0725d2d82 100644 --- a/disco/disco.go +++ b/disco/disco.go @@ -88,7 +88,14 @@ type Stator interface { // for each of its fields. type Index struct { Data []byte - Fields map[string][]byte + Fields map[string]*Field +} + +// Field is a struct which contains the data encoded for the field as well as +// for each of its views. +type Field struct { + Data []byte + Views map[string][]byte } type Schemator interface { @@ -99,6 +106,9 @@ type Schemator interface { Field(ctx context.Context, index, field string) ([]byte, error) CreateField(ctx context.Context, index, field string, val []byte) error DeleteField(ctx context.Context, index, field string) error + View(ctx context.Context, index, field, view string) ([]byte, error) + CreateView(ctx context.Context, index, field, view string, val []byte) error + DeleteView(ctx context.Context, index, field, view string) error } type Metadata interface { @@ -263,3 +273,16 @@ func (*nopSchemator) CreateField(ctx context.Context, index, field string, val [ // DeleteField is a no-op implementation of the Schemator DeleteField method. func (*nopSchemator) DeleteField(ctx context.Context, index, field string) error { return nil } + +// View is a no-op implementation of the Schemator View method. +func (*nopSchemator) View(ctx context.Context, index, field, view string) ([]byte, error) { + return nil, nil +} + +// CreateView is a no-op implementation of the Schemator CreateView method. +func (*nopSchemator) CreateView(ctx context.Context, index, field, view string, val []byte) error { + return nil +} + +// DeleteView is a no-op implementation of the Schemator DeleteView method. +func (*nopSchemator) DeleteView(ctx context.Context, index, field, view string) error { return nil } diff --git a/etcd/embed.go b/etcd/embed.go index 3caef98ca..0d2d919a4 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -484,25 +484,52 @@ func (e *Etcd) Schema(ctx context.Context) (map[string]*disco.Index, error) { return nil, err } + // The logic in the following for loop assumes that the list of keys is + // ordered such that index comes before field, which comes before view. + // For example: + // /index1 + // /index1/field1 + // /index1/field1/view1 + // /index1/field1/view2 + // /index1/field2 + // /index2 + // /index2/field1 + // m := make(map[string]*disco.Index) for i, k := range keys { tokens := strings.Split(strings.Trim(k, "/"), "/") // token[0] contains the schemaPrefix + + // token[1]: index index := tokens[1] if _, ok := m[index]; !ok { m[index] = &disco.Index{ Data: vals[i], - Fields: make(map[string][]byte), + Fields: make(map[string]*disco.Field), } + continue } flds := m[index].Fields + // token[2]: field if len(tokens) > 2 { field := tokens[2] - flds[field] = vals[i] + if _, ok := flds[field]; !ok { + flds[field] = &disco.Field{ + Data: vals[i], + Views: make(map[string][]byte), + } + continue + } + views := flds[field].Views + + // token[3]: view + if len(tokens) > 3 { + view := tokens[3] + views[view] = vals[i] + } } } - return m, nil } @@ -601,7 +628,7 @@ func (e *Etcd) Field(ctx context.Context, indexName string, name string) ([]byte func (e *Etcd) CreateField(ctx context.Context, indexName string, name string, val []byte) error { cli, err := e.client() if err != nil { - return errors.Wrap(err, "CreateIndex: creating client") + return errors.Wrap(err, "CreateField: creating client") } defer cli.Close() @@ -646,6 +673,46 @@ func (e *Etcd) DeleteField(ctx context.Context, indexname string, name string) e return errors.Wrap(err, "DeleteField") } +func (e *Etcd) View(ctx context.Context, indexName, fieldName, name string) ([]byte, error) { + key := schemaPrefix + indexName + "/" + fieldName + "/" + name + return e.getKeyBytes(ctx, key) +} + +// CreateView differs from CreateIndex and CreateField in that it does not +// return an error if the view already exists. If this logic needs to be +// changed, we likely need to introduce an ErrViewExists variable and return +// that. I decided not to do that now because it would require importing the +// etcd package into the pilosa root package. The better way to do that may be +// to define those error types in the disco package instead. +func (e *Etcd) CreateView(ctx context.Context, indexName, fieldName, name string, val []byte) error { + cli, err := e.client() + if err != nil { + return errors.Wrap(err, "CreateView: creating client") + } + defer cli.Close() + + key := schemaPrefix + indexName + "/" + fieldName + "/" + name + + // Set up Op to write view value as bytes. + op := clientv3.OpPut(key, "") + op.WithValueBytes(val) + + // Check for key existence, and execute Op within a transaction. + _, err = cli.KV.Txn(ctx). + If(clientv3util.KeyMissing(key)). + Then(op). + Commit() + if err != nil { + return errors.Wrap(err, "executing transaction") + } + + return nil +} + +func (e *Etcd) DeleteView(ctx context.Context, indexName, fieldName, name string) error { + return e.delKey(ctx, schemaPrefix+indexName+"/"+fieldName+"/"+name, false) +} + func (e *Etcd) putKey(ctx context.Context, key, val string, opts ...clientv3.OpOption) error { cli, err := e.client() if err != nil { diff --git a/field.go b/field.go index e5936503d..37a38a249 100644 --- a/field.go +++ b/field.go @@ -31,6 +31,7 @@ import ( "time" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2/disco" "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" @@ -102,6 +103,8 @@ type Field struct { broadcaster broadcaster Stats stats.StatsClient + schemator disco.Schemator + serializer Serializer // Field options. options FieldOptions @@ -367,6 +370,8 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel broadcaster: NopBroadcaster, Stats: stats.NopStatsClient, + schemator: disco.NopSchemator, + serializer: NopSerializer, options: *applyDefaultOptions(&fo), @@ -1147,19 +1152,22 @@ func (f *Field) recalculateCaches() { // createViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. func (f *Field) createViewIfNotExists(name string) (*view, error) { - view, created, err := f.createViewIfNotExistsBase(name) + cvm := &CreateViewMessage{ + Index: f.index, + Field: f.name, + View: name, + } + + // call this base method to isolate the mu.Lock and ensure we aren't holding + // the lock while calling SendSync below. + view, created, err := f.createViewIfNotExistsBase(cvm) if err != nil { return nil, err } if created { // Broadcast view creation to the cluster. - err = f.broadcaster.SendSync( - &CreateViewMessage{ - Index: f.index, - Field: f.name, - View: name, - }) + err := f.broadcaster.SendSync(cvm) if err != nil { return nil, errors.Wrap(err, "sending CreateView message") } @@ -1169,15 +1177,26 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) { } // createViewIfNotExistsBase returns the named view, creating it if necessary. -// The returned bool indicates whether the view was created or not. -func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) { +// One purpose of isolating this method from createViewIfNotExists() is that we +// need to enforce the mu.Lock on everything in this method, but we can't be +// holding the lock when broadcasting the CreateViewMessage view +// broadcaster.SendSync(); calling that SendSync() while holding the lock can +// result in a deadlock waiting on the remote node to give up its lock obtained +// by performing the same action. The returned bool indicates whether the view +// was created or not. +func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool, error) { f.mu.Lock() defer f.mu.Unlock() - if view := f.viewMap[name]; view != nil { + // Create the view in etcd as the system of record. + if err := f.persistView(context.Background(), cvm); err != nil { + return nil, false, errors.Wrap(err, "persisting view") + } + + if view := f.viewMap[cvm.View]; view != nil { return view, false, nil } - view := f.newView(f.viewPath(name), name) + view := f.newView(f.viewPath(cvm.View), cvm.View) if err := view.openEmpty(); err != nil { return nil, false, errors.Wrap(err, "opening view") @@ -1216,6 +1235,11 @@ func (f *Field) deleteView(name string) error { delete(f.viewMap, name) + // Delete the view from etcd as the system of record. + if err := f.schemator.DeleteView(context.TODO(), f.index, f.name, name); err != nil { + return errors.Wrapf(err, "deleting view from etcd: %s/%s/%s", f.index, f.name, name) + } + return nil } @@ -2200,3 +2224,21 @@ func bitDepthInt64(v int64) uint { func FormatQualifiedFieldName(index, field string) string { return fmt.Sprintf("%s\x00%s\x00", index, field) } + +// persistView stores the view information in etcd. +func (f *Field) persistView(ctx context.Context, cvm *CreateViewMessage) error { + if cvm.Index == "" { + return ErrIndexRequired + } else if cvm.Field == "" { + return ErrFieldRequired + } else if cvm.View == "" { + return ErrViewRequired + } + + if b, err := f.serializer.Marshal(cvm); err != nil { + return errors.Wrap(err, "marshaling") + } else if err := f.schemator.CreateView(ctx, cvm.Index, cvm.Field, cvm.View, b); err != nil { + return errors.Wrapf(err, "writing field to disco: %s/%s/%s", cvm.Index, cvm.Field, cvm.View) + } + return nil +} diff --git a/holder.go b/holder.go index 47345a65a..9ecf673d8 100644 --- a/holder.go +++ b/holder.go @@ -878,7 +878,7 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e return nil, errors.Wrapf(err, "getting schema via schemator") } - for indexName, index := range schema { + for _, index := range schema { cim, err := h.decodeCreateIndexMessage(index.Data) if err != nil { return nil, errors.Wrap(err, "decoding CreateIndexMessage") @@ -891,28 +891,28 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e ShardWidth: ShardWidth, Fields: make([]*FieldInfo, 0, len(index.Fields)), } - for fieldName, fieldData := range index.Fields { - createFieldMessage, err := h.decodeCreateFieldMessage(fieldData) + for _, field := range index.Fields { + cfm, err := h.decodeCreateFieldMessage(field.Data) if err != nil { return nil, errors.Wrap(err, "decoding CreateFieldMessage") } - if fieldName == existenceFieldName { + if cfm.Field == existenceFieldName { continue } fi := &FieldInfo{ - Name: fieldName, - CreatedAt: createFieldMessage.CreatedAt, - Options: *createFieldMessage.Meta, + Name: cfm.Field, + CreatedAt: cfm.CreatedAt, + Options: *cfm.Meta, } if includeViews { - // Because views are not stored in etcd, we still rely on the - // local representation of views. - if localField := h.Field(indexName, fieldName); localField != nil { - for _, view := range localField.views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) + for _, viewData := range field.Views { + cvm, err := h.decodeCreateViewMessage(viewData) + if err != nil { + return nil, errors.Wrap(err, "decoding CreateViewMessage") } - sort.Sort(viewInfoSlice(fi.Views)) + fi.Views = append(fi.Views, &ViewInfo{Name: cvm.View}) } + sort.Sort(viewInfoSlice(fi.Views)) } di.Fields = append(di.Fields, fi) } @@ -1040,16 +1040,28 @@ func (h *Holder) LoadIndex(name string) (*Index, error) { // LoadField creates a field based on the information stored in schemator. // An error is returned if the field already exists. func (h *Holder) LoadField(index, field string) (*Field, error) { - h.mu.Lock() - defer h.mu.Unlock() - // Ensure field doesn't already exist. if h.Field(index, field) != nil { return nil, newConflictError(ErrFieldExists) } + + h.mu.Lock() + defer h.mu.Unlock() + return h.loadField(index, field) } +// LoadView creates a view based on the information stored in schemator. Unlike +// index and field, it is not considered an error if the view already exists. +func (h *Holder) LoadView(index, field, view string) (*view, error) { + // If the view already exists, just return with it here. + if v := h.view(index, field, view); v != nil { + return v, nil + } + + return h.loadView(index, field, view) +} + // CreateIndexAndBroadcast creates an index locally, then broadcasts the // creation to other nodes so they can create locally as well. An error is // returned if the index already exists. @@ -1181,8 +1193,6 @@ func (h *Holder) loadIndex(indexName string) (*Index, error) { func (h *Holder) loadField(indexName, fieldName string) (*Field, error) { b, err := h.schemator.Field(context.TODO(), indexName, fieldName) if err != nil { - // TODO: we may need to wrap with ConflictError if the error type is - // ErrIndexExists. return nil, errors.Wrapf(err, "getting field: %s/%s", indexName, fieldName) } @@ -1192,12 +1202,33 @@ func (h *Holder) loadField(indexName, fieldName string) (*Field, error) { return nil, errors.Errorf("local index not found: %s", indexName) } - createFieldMessage, err := h.decodeCreateFieldMessage(b) + cfm, err := h.decodeCreateFieldMessage(b) if err != nil { return nil, errors.Wrap(err, "decoding CreateFieldMessage") } - return idx.createFieldIfNotExists(fieldName, createFieldMessage.Meta) + // TODO: can this take cfm? + return idx.createFieldIfNotExists(fieldName, cfm.Meta) +} + +func (h *Holder) loadView(indexName, fieldName, viewName string) (*view, error) { + b, err := h.schemator.View(context.TODO(), indexName, fieldName, viewName) + if err != nil { + return nil, errors.Wrapf(err, "getting view: %s/%s/%s", indexName, fieldName, viewName) + } + + // Get field. + fld := h.Field(indexName, fieldName) + if fld == nil { + return nil, errors.Errorf("local field not found: %s/%s", indexName, fieldName) + } + + cvm, err := h.decodeCreateViewMessage(b) + if err != nil { + return nil, errors.Wrap(err, "decoding CreateFieldMessage") + } + + return fld.createViewIfNotExists(cvm.View) } func (h *Holder) newIndex(path, name string) (*Index, error) { @@ -2231,3 +2262,11 @@ func (h *Holder) decodeCreateFieldMessage(b []byte) (*CreateFieldMessage, error) } return &cfm, nil } + +func (h *Holder) decodeCreateViewMessage(b []byte) (*CreateViewMessage, error) { + var cvm CreateViewMessage + if err := h.serializer.Unmarshal(b, &cvm); err != nil { + return nil, errors.Wrap(err, "unmarshaling") + } + return &cvm, nil +} diff --git a/index.go b/index.go index ac5afedc2..e510be7eb 100644 --- a/index.go +++ b/index.go @@ -102,6 +102,9 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { holder: holder, trackExistence: true, + schemator: disco.NopSchemator, + serializer: NopSerializer, + translateStores: make(map[int]TranslateStore), translationSyncer: NopTranslationSyncer, @@ -523,7 +526,7 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { // Create the field in etcd as the system of record. if err := i.persistField(context.Background(), cfm); err != nil { - return nil, errors.Wrap(err, "persisting index") + return nil, errors.Wrap(err, "persisting field") } return i.createField(cfm, false) @@ -548,7 +551,7 @@ func (i *Index) CreateFieldAndBroadcast(cfm *CreateFieldMessage) (*Field, error) // Create the field in etcd as the system of record. if err := i.persistField(context.Background(), cfm); err != nil { - return nil, errors.Wrap(err, "persisting index") + return nil, errors.Wrap(err, "persisting field") } return i.createField(cfm, true) @@ -588,7 +591,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field // persistent storage. In that case, this will return an "index exists" // error, which in that case should return the index. TODO: We may need // to allow for that in the future. - return nil, errors.Wrap(err, "persisting index") + return nil, errors.Wrap(err, "persisting field") } return i.createField(cfm, false) @@ -697,6 +700,8 @@ func (i *Index) newField(path, name string) (*Field, error) { f.idx = i f.Stats = i.Stats f.broadcaster = i.broadcaster + f.schemator = i.schemator + f.serializer = i.serializer f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) f.OpenTranslateStore = i.OpenTranslateStore return f, nil diff --git a/pilosa.go b/pilosa.go index c98f886e0..dbd4e6567 100644 --- a/pilosa.go +++ b/pilosa.go @@ -53,6 +53,8 @@ var ( ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrDecimalOutOfRange = errors.New("decimal value out of range") + ErrViewRequired = errors.New("view required") + ErrViewExists = errors.New("view already exists") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/server.go b/server.go index 800d4bb8c..216444a0e 100644 --- a/server.go +++ b/server.go @@ -415,12 +415,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { metricInterval: 0, diagnosticInterval: 0, - disCo: disco.NopDisCo, - stator: disco.NopStator, - metadator: disco.NopMetadator, - resizer: disco.NopResizer, - noder: topology.NewEmptyLocalNoder(), - sharder: disco.NopSharder, + disCo: disco.NopDisCo, + stator: disco.NopStator, + metadator: disco.NopMetadator, + resizer: disco.NopResizer, + noder: topology.NewEmptyLocalNoder(), + sharder: disco.NopSharder, + schemator: disco.NopSchemator, + serializer: NopSerializer, confirmDownRetries: defaultConfirmDownRetries, confirmDownSleep: defaultConfirmDownSleep, @@ -807,11 +809,7 @@ func (s *Server) receiveMessage(m Message) error { } case *CreateViewMessage: - f := s.holder.Field(obj.Index, obj.Field) - if f == nil { - return fmt.Errorf("local field not found: %s", obj.Field) - } - if _, _, err := f.createViewIfNotExistsBase(obj.View); err != nil { + if _, err := s.holder.LoadView(obj.Index, obj.Field, obj.View); err != nil { return err } From bcb71b023a6ed4ca692a8f9a3855e90eb2122f74 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 9 Feb 2021 10:49:33 -0600 Subject: [PATCH 2/4] fix misplaced _exists check --- holder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/holder.go b/holder.go index 9ecf673d8..c1d1855e7 100644 --- a/holder.go +++ b/holder.go @@ -891,14 +891,14 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e ShardWidth: ShardWidth, Fields: make([]*FieldInfo, 0, len(index.Fields)), } - for _, field := range index.Fields { + for fieldName, field := range index.Fields { + if fieldName == existenceFieldName { + continue + } cfm, err := h.decodeCreateFieldMessage(field.Data) if err != nil { return nil, errors.Wrap(err, "decoding CreateFieldMessage") } - if cfm.Field == existenceFieldName { - continue - } fi := &FieldInfo{ Name: cfm.Field, CreatedAt: cfm.CreatedAt, From 1c3b0f364d4071e2f01a99dfca912718271ba60a Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 9 Feb 2021 23:23:00 -0600 Subject: [PATCH 3/4] implement ApplySchema, LoadSchema, and LoadSchemaMessage --- api.go | 27 +--- broadcast.go | 5 + encoding/proto/proto.go | 17 +++ holder.go | 82 +++++++--- index.go | 38 +++++ internal/private.pb.go | 327 ++++++++++++++++++++++++++++------------ internal/private.proto | 2 + server.go | 3 + 8 files changed, 364 insertions(+), 137 deletions(-) diff --git a/api.go b/api.go index c5a815c8b..929611681 100644 --- a/api.go +++ b/api.go @@ -1011,31 +1011,12 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } - // set CreatedAt for indexes and fields (if empty), and then apply schema. - for _, index := range s.Indexes { - if index.CreatedAt == 0 { - index.CreatedAt = timestamp() - } - for _, field := range index.Fields { - if field.CreatedAt == 0 { - field.CreatedAt = timestamp() - } - } - } - if !remote { - nodes := api.cluster.Nodes() - for i, node := range nodes { - if node.ID == api.Node().ID { - continue - } - err := api.server.defaultClient.PostSchema(ctx, &node.URI, s, true) - if err != nil { - return errors.Wrapf(err, "forwarding post schema to node %d of %d", i+1, len(nodes)) - } - } + err := api.holder.applySchema(s) + if err != nil { + return errors.Wrap(err, "applying schema") } - return errors.Wrap(api.holder.applySchema(s), "applying schema") + return nil } // Views returns the views in the given field. diff --git a/broadcast.go b/broadcast.go index 8ed654b7b..141c43e36 100644 --- a/broadcast.go +++ b/broadcast.go @@ -77,6 +77,7 @@ const ( messageTypeResizeInstructionComplete messageTypeNodeState messageTypeRecalculateCaches + messageTypeLoadSchemaMessage messageTypeNodeEvent messageTypeNodeStatus messageTypeTransaction @@ -121,6 +122,8 @@ func getMessage(typ byte) Message { return &NodeStateMessage{} case messageTypeRecalculateCaches: return &RecalculateCaches{} + case messageTypeLoadSchemaMessage: + return &LoadSchemaMessage{} case messageTypeNodeEvent: return &NodeEvent{} case messageTypeNodeStatus: @@ -162,6 +165,8 @@ func getMessageType(m Message) byte { return messageTypeNodeState case *RecalculateCaches: return messageTypeRecalculateCaches + case *LoadSchemaMessage: + return messageTypeLoadSchemaMessage case *NodeEvent: return messageTypeNodeEvent case *NodeStatus: diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 7785a6f26..7226dce7f 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -154,6 +154,14 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } s.decodeRecalculateCaches(msg, mt) return nil + case *pilosa.LoadSchemaMessage: + msg := &internal.LoadSchemaMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling LoadSchemaMessage") + } + s.decodeLoadSchemaMessage(msg, mt) + return nil case *pilosa.NodeEvent: msg := &internal.NodeEventMessage{} err := proto.Unmarshal(buf, msg) @@ -358,6 +366,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeNodeStateMessage(mt) case *pilosa.RecalculateCaches: return s.encodeRecalculateCaches(mt) + case *pilosa.LoadSchemaMessage: + return s.encodeLoadSchemaMessage(mt) case *pilosa.NodeEvent: return s.encodeNodeEventMessage(mt) case *pilosa.NodeStatus: @@ -855,6 +865,10 @@ func (s Serializer) encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal return &internal.RecalculateCaches{} } +func (s Serializer) encodeLoadSchemaMessage(*pilosa.LoadSchemaMessage) *internal.LoadSchemaMessage { + return &internal.LoadSchemaMessage{} +} + func (s Serializer) encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest { return &internal.TranslateKeysRequest{ Index: request.Index, @@ -1185,6 +1199,9 @@ func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldS func (s Serializer) decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) { } +func (s Serializer) decodeLoadSchemaMessage(pb *internal.LoadSchemaMessage, m *pilosa.LoadSchemaMessage) { +} + func (s Serializer) decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { m.Query = pb.Query m.Shards = pb.Shards diff --git a/holder.go b/holder.go index c1d1855e7..9b81591a0 100644 --- a/holder.go +++ b/holder.go @@ -925,29 +925,21 @@ func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, e // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { - // Create indexes that don't exist. + // Create indexes. + // We use h.CreateIndex() instead of h.CreateIndexIfNotExists() because we + // want to limit the use of this method for now to only new indexes. for _, i := range schema.Indexes { - idx, err := h.CreateIndexIfNotExists(i.Name, i.Options) + idx, err := h.CreateIndex(i.Name, i.Options) if err != nil { return errors.Wrap(err, "creating index") } - if i.CreatedAt != 0 { - idx.mu.Lock() - idx.createdAt = i.CreatedAt - idx.mu.Unlock() - } // Create fields that don't exist. for _, f := range i.Fields { - fld, err := idx.createFieldIfNotExists(f.Name, &f.Options) + fld, err := idx.CreateFieldIfNotExistsWithOptions(f.Name, &f.Options) if err != nil { return errors.Wrap(err, "creating field") } - if f.CreatedAt != 0 { - fld.mu.Lock() - fld.createdAt = f.CreatedAt - fld.mu.Unlock() - } // Create views that don't exist. for _, v := range f.Views { @@ -958,6 +950,12 @@ func (h *Holder) applySchema(schema *Schema) error { } } } + + // Send the load schema message to all nodes. + if err := h.broadcaster.SendSync(&LoadSchemaMessage{}); err != nil { + return errors.Wrap(err, "sending LoadSchemaMessage") + } + return nil } @@ -1012,7 +1010,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { cim := &CreateIndexMessage{ Index: name, - CreatedAt: 0, + CreatedAt: timestamp(), Meta: &opt, } @@ -1024,6 +1022,22 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { return h.createIndex(cim, false) } +// LoadSchemaMessage is an internal message used to inform a node to load the +// latest schema from etcd. +type LoadSchemaMessage struct{} + +// LoadSchema creates all indexes based on the information stored in schemator. +// It does not return an error if an index already exists. The thinking is that +// this method will load all indexes that don't already exist. We likely want to +// revisit this; for example, we might want to confirm that the createdAt +// timestamps on each of the indexes matches the value in etcd. +func (h *Holder) LoadSchema() error { + h.mu.Lock() + defer h.mu.Unlock() + + return h.loadSchema() +} + // LoadIndex creates an index based on the information stored in schemator. // An error is returned if the index already exists. func (h *Holder) LoadIndex(name string) (*Index, error) { @@ -1091,7 +1105,7 @@ func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, cim := &CreateIndexMessage{ Index: name, - CreatedAt: 0, + CreatedAt: timestamp(), Meta: &opt, } @@ -1174,11 +1188,45 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e return index, nil } +func (h *Holder) loadSchema() error { + schema, err := h.schemator.Schema(context.TODO()) + if err != nil { + return errors.Wrap(err, "getting schema") + } + + // TODO: This is kind of inefficient because we're ignoring the index.Data + // and field.Data values, which contains the index and field information, + // and only using the map key to call loadIndex() and loadField(). These + // make another call to schemator to get the same index and field + // information that we already have in the map. It probably makes sense to + // either copy the parts of the loadIndex and loadField methods here (like + // decodeCreateIndexMessage) or split loadIndex and loadField into smaller + // methods that we could reuse here. + for indexName, index := range schema { + _, err := h.loadIndex(indexName) + if err != nil { + return errors.Wrap(err, "loading index") + } + for fieldName, field := range index.Fields { + _, err := h.loadField(indexName, fieldName) + if err != nil { + return errors.Wrap(err, "loading field") + } + for viewName := range field.Views { + _, err := h.loadView(indexName, fieldName, viewName) + if err != nil { + return errors.Wrap(err, "loading view") + } + } + } + } + + return nil +} + func (h *Holder) loadIndex(indexName string) (*Index, error) { b, err := h.schemator.Index(context.TODO(), indexName) if err != nil { - // TODO: we may need to wrap with ConflictError if the error type is - // ErrIndexExists. return nil, errors.Wrapf(err, "getting index: %s", indexName) } diff --git a/index.go b/index.go index e510be7eb..83d5d8c44 100644 --- a/index.go +++ b/index.go @@ -597,6 +597,44 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field return i.createField(cfm, false) } +// CreateFieldIfNotExistsWithOptions is a method which I created because I +// needed the functionality of CreateFieldIfNotExists, but instead of taking +// function options, taking a *FieldOptions struct. TODO: This should +// definintely be refactored so we don't have these virtually equivalent +// methods, but I'm puttin this here for now just to see if it works. +func (i *Index) CreateFieldIfNotExistsWithOptions(name string, opt *FieldOptions) (*Field, error) { + err := validateName(name) + if err != nil { + return nil, errors.Wrap(err, "validating name") + } + + i.mu.Lock() + defer i.mu.Unlock() + + // Find field in cache first. + if f := i.fields[name]; f != nil { + return f, nil + } + + cfm := &CreateFieldMessage{ + Index: i.name, + Field: name, + CreatedAt: 0, + Meta: opt, + } + + // Create the field in etcd as the system of record. + if err := i.persistField(context.Background(), cfm); err != nil { + // There is a case where the index is not in memory, but it is in + // persistent storage. In that case, this will return an "index exists" + // error, which in that case should return the index. TODO: We may need + // to allow for that in the future. + return nil, errors.Wrap(err, "persisting field") + } + + return i.createField(cfm, false) +} + // persistField stores the field information in etcd. func (i *Index) persistField(ctx context.Context, cfm *CreateFieldMessage) error { if cfm.Index == "" { diff --git a/internal/private.pb.go b/internal/private.pb.go index 08fe39847..742211a43 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -2157,6 +2157,45 @@ func (m *RecalculateCaches) XXX_DiscardUnknown() { var xxx_messageInfo_RecalculateCaches proto.InternalMessageInfo +type LoadSchemaMessage struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadSchemaMessage) Reset() { *m = LoadSchemaMessage{} } +func (m *LoadSchemaMessage) String() string { return proto.CompactTextString(m) } +func (*LoadSchemaMessage) ProtoMessage() {} +func (*LoadSchemaMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{33} +} +func (m *LoadSchemaMessage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *LoadSchemaMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_LoadSchemaMessage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *LoadSchemaMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadSchemaMessage.Merge(m, src) +} +func (m *LoadSchemaMessage) XXX_Size() int { + return m.Size() +} +func (m *LoadSchemaMessage) XXX_DiscardUnknown() { + xxx_messageInfo_LoadSchemaMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadSchemaMessage proto.InternalMessageInfo + type TransactionMessage struct { Action string `protobuf:"bytes,1,opt,name=Action,proto3" json:"Action,omitempty"` Transaction *Transaction `protobuf:"bytes,2,opt,name=Transaction,proto3" json:"Transaction,omitempty"` @@ -2169,7 +2208,7 @@ func (m *TransactionMessage) Reset() { *m = TransactionMessage{} } func (m *TransactionMessage) String() string { return proto.CompactTextString(m) } func (*TransactionMessage) ProtoMessage() {} func (*TransactionMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{33} + return fileDescriptor_d2a91b51c7bdc125, []int{34} } func (m *TransactionMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2228,7 +2267,7 @@ func (m *Transaction) Reset() { *m = Transaction{} } func (m *Transaction) String() string { return proto.CompactTextString(m) } func (*Transaction) ProtoMessage() {} func (*Transaction) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{34} + return fileDescriptor_d2a91b51c7bdc125, []int{35} } func (m *Transaction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2309,7 +2348,7 @@ func (m *TransactionStats) Reset() { *m = TransactionStats{} } func (m *TransactionStats) String() string { return proto.CompactTextString(m) } func (*TransactionStats) ProtoMessage() {} func (*TransactionStats) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{35} + return fileDescriptor_d2a91b51c7bdc125, []int{36} } func (m *TransactionStats) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2348,7 +2387,7 @@ func (m *ResizeAbortMessage) Reset() { *m = ResizeAbortMessage{} } func (m *ResizeAbortMessage) String() string { return proto.CompactTextString(m) } func (*ResizeAbortMessage) ProtoMessage() {} func (*ResizeAbortMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{36} + return fileDescriptor_d2a91b51c7bdc125, []int{37} } func (m *ResizeAbortMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2389,7 +2428,7 @@ func (m *ResizeNodeMessage) Reset() { *m = ResizeNodeMessage{} } func (m *ResizeNodeMessage) String() string { return proto.CompactTextString(m) } func (*ResizeNodeMessage) ProtoMessage() {} func (*ResizeNodeMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_d2a91b51c7bdc125, []int{37} + return fileDescriptor_d2a91b51c7bdc125, []int{38} } func (m *ResizeNodeMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2467,6 +2506,7 @@ func init() { proto.RegisterType((*ResizeInstructionComplete)(nil), "internal.ResizeInstructionComplete") proto.RegisterType((*Topology)(nil), "internal.Topology") proto.RegisterType((*RecalculateCaches)(nil), "internal.RecalculateCaches") + proto.RegisterType((*LoadSchemaMessage)(nil), "internal.LoadSchemaMessage") proto.RegisterType((*TransactionMessage)(nil), "internal.TransactionMessage") proto.RegisterType((*Transaction)(nil), "internal.Transaction") proto.RegisterType((*TransactionStats)(nil), "internal.TransactionStats") @@ -2477,98 +2517,98 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1446 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0x66, 0xbd, 0x76, 0x6c, 0x1f, 0xc7, 0xa9, 0x33, 0x4d, 0xd3, 0x6d, 0xa8, 0x82, 0x19, 0x10, - 0x35, 0x95, 0x1a, 0xaa, 0x16, 0x09, 0x04, 0xaa, 0xd4, 0x24, 0x4e, 0x8b, 0x81, 0xb4, 0xe9, 0x24, - 0xed, 0xfd, 0x64, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0xba, 0xfb, 0x93, 0xda, 0x45, 0xe2, 0x16, 0x04, - 0x57, 0x08, 0x2e, 0xb8, 0xe4, 0x3d, 0x78, 0x01, 0x2e, 0x79, 0x04, 0x54, 0x9e, 0x80, 0x37, 0x40, - 0x73, 0x66, 0x66, 0x77, 0xed, 0x38, 0x75, 0x68, 0xb9, 0xdb, 0xf3, 0xff, 0x9d, 0x9f, 0x39, 0x33, - 0x36, 0x34, 0x87, 0x91, 0x77, 0xc2, 0x13, 0xb1, 0x31, 0x8c, 0xc2, 0x24, 0x24, 0x35, 0x2f, 0x48, - 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x71, 0x98, 0x1e, 0xfa, 0x9e, 0xab, 0xf8, 0xf4, 0x3e, 0xd4, 0x7b, - 0x41, 0x5f, 0x8c, 0x76, 0x45, 0xc2, 0x09, 0x81, 0xf2, 0x57, 0x62, 0x1c, 0x3b, 0x76, 0xdb, 0xea, - 0xd4, 0x18, 0x7e, 0x93, 0x0f, 0x60, 0xe9, 0x20, 0xe2, 0xee, 0xf1, 0xce, 0xc8, 0x8b, 0x13, 0x11, - 0xb8, 0xc2, 0x29, 0xa3, 0x74, 0x8a, 0x4b, 0x7f, 0xb3, 0x61, 0xf1, 0x9e, 0x27, 0xfc, 0xfe, 0xc3, - 0x61, 0xe2, 0x85, 0x41, 0x2c, 0x9d, 0x1d, 0x8c, 0x87, 0xc2, 0xa9, 0xb5, 0xad, 0x4e, 0x9d, 0xe1, - 0x37, 0xb9, 0x0a, 0xf5, 0x6d, 0xee, 0x1e, 0x09, 0x14, 0xd8, 0x28, 0xc8, 0x19, 0x99, 0x74, 0xdf, - 0x7b, 0xa1, 0xa2, 0x34, 0x59, 0xce, 0x20, 0x6d, 0x68, 0x1c, 0x78, 0x03, 0xf1, 0x28, 0xe5, 0x41, - 0x92, 0x0e, 0x9c, 0x0a, 0x5a, 0x17, 0x59, 0x64, 0x15, 0x16, 0x1e, 0xfa, 0xfd, 0x5d, 0x2f, 0x70, - 0xea, 0x6d, 0xab, 0x63, 0x33, 0x4d, 0x19, 0x3e, 0x1f, 0x39, 0x90, 0xf3, 0xf9, 0x28, 0x4b, 0xb7, - 0x31, 0x99, 0xee, 0x83, 0x70, 0x3f, 0xe1, 0x41, 0x9f, 0x47, 0xfd, 0x27, 0x9e, 0x78, 0xee, 0x2c, - 0xaa, 0x74, 0x27, 0xb9, 0xd2, 0x76, 0x8b, 0xc7, 0xc2, 0x69, 0xa2, 0x47, 0xfc, 0x26, 0x6b, 0x50, - 0xdb, 0xf2, 0x92, 0xae, 0x18, 0x26, 0x47, 0xce, 0x52, 0xdb, 0xea, 0x94, 0x59, 0x46, 0x93, 0x15, - 0xa8, 0xec, 0xbb, 0xdc, 0x17, 0xce, 0x05, 0x34, 0x50, 0x04, 0xa1, 0xb0, 0x78, 0x2f, 0x8c, 0x84, - 0xf7, 0x34, 0xc0, 0x26, 0x38, 0x2d, 0x4c, 0x6a, 0x82, 0x47, 0xde, 0x03, 0x5b, 0xa6, 0xb4, 0xdc, - 0xb6, 0x3a, 0x8d, 0x5b, 0xcb, 0x1b, 0xa6, 0x8f, 0x1b, 0x5d, 0xe1, 0x7a, 0x03, 0xee, 0x33, 0x29, - 0x45, 0x25, 0x3e, 0x72, 0xc8, 0xd9, 0x4a, 0x7c, 0x44, 0x29, 0x2c, 0xf5, 0x06, 0xc3, 0x30, 0x4a, - 0x98, 0x88, 0x87, 0x61, 0x10, 0x0b, 0xd2, 0x02, 0x7b, 0x27, 0x8a, 0x1c, 0x0b, 0xc3, 0xca, 0x4f, - 0xfa, 0x2d, 0xb4, 0xb6, 0xfc, 0xd0, 0x3d, 0xee, 0xf2, 0x84, 0x33, 0xf1, 0x2c, 0x15, 0x71, 0x22, - 0xb1, 0x2b, 0x78, 0x4a, 0x4f, 0x11, 0x92, 0x8b, 0xfd, 0x76, 0x4a, 0x8a, 0x8b, 0x84, 0xac, 0x0b, - 0x56, 0x4d, 0xb5, 0x07, 0xbf, 0x31, 0xf7, 0x23, 0x1e, 0xf5, 0xb1, 0xa7, 0x65, 0xa6, 0x08, 0xc9, - 0xc5, 0x48, 0x38, 0x07, 0x65, 0xa6, 0x08, 0xda, 0x83, 0xe5, 0x42, 0x7c, 0x0d, 0x73, 0x15, 0x16, - 0x58, 0xf8, 0xbc, 0xd7, 0x8d, 0x1d, 0xab, 0x6d, 0x77, 0xca, 0x4c, 0x53, 0x38, 0x30, 0xa1, 0x9f, - 0x0e, 0x02, 0x29, 0x2a, 0xa1, 0x28, 0x67, 0xd0, 0x2b, 0x50, 0xc1, 0xe9, 0x91, 0x59, 0xe6, 0xb6, - 0xf2, 0x93, 0x7e, 0x67, 0x41, 0x7d, 0x97, 0x8f, 0x10, 0x48, 0x4c, 0xee, 0x40, 0xcd, 0xf4, 0x16, - 0x95, 0x1a, 0xb7, 0xde, 0xcd, 0x2b, 0x98, 0xa9, 0x6d, 0x18, 0x9d, 0x9d, 0x20, 0x89, 0xc6, 0x2c, - 0x33, 0x59, 0xfb, 0x1c, 0x9a, 0x13, 0x22, 0x19, 0xef, 0x58, 0x8c, 0x4d, 0x55, 0x8f, 0xc5, 0x58, - 0xe6, 0x7a, 0xc2, 0xfd, 0x54, 0x60, 0xad, 0xca, 0x4c, 0x11, 0x9f, 0x95, 0x3e, 0xb5, 0xe8, 0x13, - 0x20, 0xdb, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0xbb, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf, 0xe2, 0x76, - 0xb1, 0xe2, 0x59, 0x75, 0x4b, 0x85, 0xea, 0xd2, 0xeb, 0x40, 0xba, 0xc2, 0x17, 0x89, 0xd0, 0xa7, - 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x06, 0x65, 0xb9, 0x2a, 0x30, 0x58, - 0xe3, 0xd6, 0xc5, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xfd, 0xcd, 0x04, - 0x01, 0xdb, 0x2c, 0x67, 0xd0, 0x1f, 0x2c, 0x13, 0x13, 0x93, 0x38, 0x67, 0xde, 0x13, 0x93, 0x76, - 0x5d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0x29, 0x4f, 0x83, 0xb9, - 0x6b, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0x6f, 0x2b, 0x0f, 0x9b, 0x27, 0xdc, 0xf3, 0xf9, 0xa1, - 0xff, 0x9f, 0xda, 0x39, 0x91, 0x96, 0x03, 0x55, 0xb4, 0xed, 0x75, 0xf5, 0xc1, 0x30, 0x24, 0xfd, - 0x06, 0xf2, 0x33, 0xf6, 0x80, 0x0f, 0x84, 0xf6, 0x86, 0xdf, 0x59, 0x35, 0x4a, 0xe7, 0xa8, 0xc6, - 0x0a, 0x54, 0xe4, 0xb9, 0x94, 0x7b, 0xde, 0x96, 0x81, 0x91, 0x98, 0x53, 0xa3, 0xdb, 0xb0, 0xb0, - 0xef, 0x1e, 0x89, 0x01, 0x27, 0x1f, 0x42, 0x15, 0xf1, 0x8b, 0x58, 0x1f, 0x96, 0x0b, 0x53, 0x43, - 0xc0, 0x8c, 0x9c, 0xfe, 0x64, 0xe9, 0xc4, 0x67, 0x42, 0x9e, 0x08, 0x58, 0x9a, 0x0a, 0x48, 0x6e, - 0x40, 0x55, 0xa3, 0xc6, 0x5d, 0x72, 0xc6, 0xac, 0x19, 0x1d, 0x72, 0x0d, 0x16, 0x30, 0xd3, 0xd8, - 0x29, 0x4f, 0x83, 0x42, 0x3e, 0xd3, 0x62, 0xba, 0x03, 0xf6, 0x63, 0xd6, 0x93, 0x2b, 0x05, 0xf3, - 0x31, 0x90, 0x34, 0x25, 0x81, 0x7e, 0x11, 0xc6, 0x89, 0xee, 0x09, 0x7e, 0x4b, 0xde, 0x5e, 0x18, - 0xa9, 0x29, 0x6e, 0x32, 0xfc, 0xa6, 0xbf, 0x58, 0x50, 0x7e, 0x10, 0xf6, 0x05, 0x59, 0x82, 0x52, - 0xaf, 0xab, 0x9d, 0x94, 0x7a, 0x5d, 0xf2, 0x0e, 0xfa, 0xd7, 0x7d, 0x68, 0xe6, 0x28, 0x1e, 0xb3, - 0x1e, 0xc3, 0xc8, 0x57, 0xa1, 0xde, 0x8b, 0xf7, 0x22, 0x6f, 0xc0, 0xa3, 0xb1, 0xbe, 0x69, 0x73, - 0x06, 0x9e, 0xe6, 0x84, 0x27, 0xea, 0xfe, 0xab, 0x33, 0x45, 0x90, 0x6b, 0x50, 0xbd, 0xcf, 0xf6, - 0xb6, 0xa5, 0xe3, 0xca, 0x2c, 0xc7, 0x46, 0x4a, 0xef, 0x42, 0x4b, 0xa2, 0x42, 0x2b, 0x33, 0x7d, - 0xab, 0xb0, 0x20, 0x79, 0x19, 0x4a, 0x4d, 0xe5, 0xa1, 0x4a, 0x85, 0x50, 0xf4, 0x6b, 0xe5, 0x61, - 0xe7, 0x44, 0x04, 0x49, 0x61, 0x7e, 0x91, 0x46, 0x07, 0x4d, 0xa6, 0x08, 0x42, 0x55, 0x05, 0x74, - 0xaa, 0x4b, 0x39, 0x22, 0xc9, 0x65, 0x28, 0xa3, 0x3f, 0x5a, 0x00, 0x06, 0x50, 0x1a, 0x67, 0x26, - 0xd6, 0xd9, 0x26, 0xa4, 0x63, 0x26, 0x4d, 0x9f, 0xec, 0x56, 0xae, 0xa5, 0xf8, 0xcc, 0x4c, 0xe2, - 0x47, 0xf9, 0x24, 0xaa, 0xa6, 0x5f, 0x9a, 0x1a, 0x11, 0x15, 0x35, 0x9f, 0xc7, 0x00, 0x1a, 0x05, - 0xfe, 0xcc, 0xa1, 0xbc, 0x91, 0xcd, 0x51, 0x69, 0xda, 0x25, 0xf2, 0xb5, 0x4b, 0xad, 0x34, 0x67, - 0xcb, 0x79, 0xd0, 0x28, 0x18, 0xcd, 0x8c, 0xd7, 0x81, 0x0b, 0x93, 0x3b, 0xc3, 0x5c, 0x64, 0xd3, - 0xec, 0x39, 0xa1, 0x7e, 0xb6, 0xa0, 0xb9, 0xed, 0xa7, 0x71, 0x22, 0x22, 0x1d, 0x4d, 0xea, 0x2b, - 0x46, 0xd6, 0xf9, 0x9c, 0x31, 0xbb, 0xf9, 0xe4, 0x7d, 0xa8, 0xc8, 0x1e, 0xa8, 0xcd, 0x70, 0xba, - 0x41, 0x4a, 0x58, 0xe8, 0x50, 0xf9, 0xd5, 0x1d, 0xa2, 0x4f, 0xa0, 0xb6, 0xb5, 0xdf, 0xbb, 0x1f, - 0x85, 0xe9, 0x70, 0x66, 0xf6, 0xe6, 0x8d, 0x58, 0x2a, 0xbc, 0x11, 0x5b, 0xea, 0xbd, 0xa3, 0x32, - 0xc4, 0xc7, 0x4d, 0x4b, 0x3d, 0x6e, 0xca, 0x9a, 0xc3, 0x47, 0x74, 0x1f, 0x96, 0x55, 0xea, 0x72, - 0x75, 0xbd, 0xce, 0x96, 0x35, 0xcf, 0x14, 0x3b, 0x7f, 0xa6, 0x48, 0xa7, 0x6a, 0x89, 0xff, 0x9f, - 0x4e, 0xff, 0x29, 0xc1, 0x32, 0x13, 0xb1, 0xf7, 0x42, 0xf4, 0x82, 0x38, 0x89, 0x52, 0x57, 0xae, - 0x2b, 0x69, 0xff, 0x65, 0x78, 0xa8, 0xfb, 0x62, 0x33, 0x45, 0x9c, 0xe7, 0x40, 0x91, 0x0e, 0x54, - 0x8b, 0xbb, 0xe3, 0xb4, 0x9a, 0x11, 0x93, 0x9b, 0x50, 0xdd, 0x0f, 0xd3, 0xc8, 0xcd, 0x4e, 0x47, - 0xe1, 0x52, 0x50, 0x88, 0x94, 0x98, 0x19, 0x35, 0xf2, 0x08, 0xc8, 0x41, 0xc4, 0x83, 0xd8, 0xe7, - 0x12, 0xa4, 0x31, 0xae, 0x4d, 0xbf, 0x88, 0x0a, 0x3a, 0x13, 0x7e, 0x66, 0x18, 0x93, 0x8f, 0x8b, - 0xc7, 0xdf, 0xa9, 0x22, 0xe2, 0x95, 0x49, 0xc4, 0xfa, 0x44, 0x15, 0xd7, 0xc4, 0x9d, 0xa9, 0x59, - 0x76, 0x16, 0xd0, 0xf0, 0x72, 0x6e, 0x38, 0x21, 0x66, 0x93, 0xda, 0xf4, 0x7b, 0x0b, 0x16, 0x8b, - 0xc8, 0xce, 0xb5, 0x76, 0xb2, 0x46, 0x97, 0xe6, 0x3f, 0xb9, 0x4c, 0xa3, 0xcb, 0xb3, 0x1e, 0xb9, - 0x95, 0xe2, 0x33, 0x2c, 0x85, 0xcb, 0x67, 0x94, 0xeb, 0x0d, 0x40, 0xb5, 0xa1, 0xb1, 0xc7, 0xa3, - 0xc4, 0x93, 0x2e, 0xf5, 0x33, 0xa1, 0xc2, 0x8a, 0x2c, 0x7a, 0x0c, 0x57, 0x4e, 0x0d, 0xdd, 0x76, - 0x38, 0x18, 0xca, 0xe9, 0x7e, 0x83, 0xe1, 0x93, 0xf7, 0x40, 0x14, 0x85, 0x91, 0xa9, 0x06, 0x12, - 0x74, 0x0b, 0x6a, 0x07, 0xe1, 0x30, 0xf4, 0xc3, 0xa7, 0xe3, 0x39, 0x4b, 0xc7, 0x81, 0xaa, 0xba, - 0x7b, 0xd4, 0x92, 0xab, 0x33, 0x43, 0xd2, 0x8b, 0xf2, 0x94, 0xb8, 0xdc, 0x77, 0x53, 0x9f, 0x27, - 0x02, 0x9f, 0xed, 0x31, 0x15, 0x7a, 0x1e, 0x39, 0xe2, 0x2f, 0x5c, 0x67, 0x9b, 0xc8, 0x30, 0xd7, - 0x99, 0xa2, 0xc8, 0x27, 0xd0, 0x28, 0x68, 0xeb, 0x3c, 0x2e, 0x4d, 0x8d, 0xad, 0x12, 0xb2, 0xa2, - 0x26, 0xfd, 0xdd, 0x9a, 0xb0, 0x3c, 0x75, 0xa3, 0xeb, 0x80, 0x27, 0xaa, 0x36, 0x35, 0xa6, 0x29, - 0x99, 0xeb, 0xce, 0xc8, 0xf5, 0xd3, 0x58, 0x8a, 0xf4, 0x45, 0x9e, 0x31, 0x64, 0xae, 0xf2, 0xb7, - 0x69, 0x98, 0x9a, 0xc7, 0x94, 0x21, 0xe5, 0xcf, 0xc4, 0xae, 0xe0, 0x7d, 0xdf, 0x0b, 0x04, 0x0e, - 0x8b, 0xcd, 0x32, 0x9a, 0xdc, 0x54, 0x6b, 0xd9, 0x4c, 0xfc, 0xda, 0x4c, 0xf8, 0xa8, 0xa1, 0x56, - 0x76, 0x4c, 0x09, 0xb4, 0xa6, 0x45, 0x74, 0x05, 0x88, 0x6a, 0xff, 0xe6, 0x61, 0x18, 0x99, 0x5b, - 0x9c, 0x6e, 0x9b, 0x4d, 0x24, 0x8b, 0x3e, 0xef, 0x71, 0x90, 0x57, 0xb9, 0x54, 0xac, 0xf2, 0x56, - 0xeb, 0x8f, 0x97, 0xeb, 0xd6, 0x9f, 0x2f, 0xd7, 0xad, 0xbf, 0x5e, 0xae, 0x5b, 0xbf, 0xfe, 0xbd, - 0xfe, 0xd6, 0xe1, 0x02, 0xfe, 0x91, 0x70, 0xfb, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x59, 0x5b, - 0x70, 0x29, 0x71, 0x10, 0x00, 0x00, + // 1456 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xc5, + 0x17, 0xff, 0xaf, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x75, 0xa6, 0x69, 0xba, 0xcd, 0xbf, 0x0a, 0x66, + 0x40, 0xd4, 0x54, 0x6a, 0xa8, 0x5a, 0x24, 0x10, 0xa8, 0x52, 0x93, 0x38, 0x2d, 0x86, 0xa6, 0x4d, + 0x27, 0x69, 0xef, 0x27, 0xeb, 0x51, 0xb3, 0xca, 0x7a, 0xd7, 0xdd, 0x8f, 0xd4, 0x2e, 0x12, 0xb7, + 0x20, 0xb8, 0x42, 0x70, 0xc1, 0x25, 0xef, 0xc1, 0x0b, 0x70, 0xc9, 0x23, 0xa0, 0xf2, 0x04, 0xbc, + 0x01, 0x9a, 0x33, 0x33, 0xbb, 0x6b, 0xc7, 0xa9, 0x43, 0xcb, 0xdd, 0x9e, 0xef, 0xdf, 0xf9, 0x98, + 0x33, 0x63, 0x43, 0x73, 0x18, 0x79, 0x27, 0x3c, 0x11, 0x1b, 0xc3, 0x28, 0x4c, 0x42, 0x52, 0xf3, + 0x82, 0x44, 0x44, 0x01, 0xf7, 0xd7, 0x16, 0x87, 0xe9, 0xa1, 0xef, 0xb9, 0x8a, 0x4f, 0xef, 0x43, + 0xbd, 0x17, 0xf4, 0xc5, 0x68, 0x57, 0x24, 0x9c, 0x10, 0x28, 0x7f, 0x25, 0xc6, 0xb1, 0x63, 0xb7, + 0xad, 0x4e, 0x8d, 0xe1, 0x37, 0xf9, 0x00, 0x96, 0x0e, 0x22, 0xee, 0x1e, 0xef, 0x8c, 0xbc, 0x38, + 0x11, 0x81, 0x2b, 0x9c, 0x32, 0x4a, 0xa7, 0xb8, 0xf4, 0x57, 0x1b, 0x16, 0xef, 0x79, 0xc2, 0xef, + 0x3f, 0x1a, 0x26, 0x5e, 0x18, 0xc4, 0xd2, 0xd9, 0xc1, 0x78, 0x28, 0x9c, 0x5a, 0xdb, 0xea, 0xd4, + 0x19, 0x7e, 0x93, 0xab, 0x50, 0xdf, 0xe6, 0xee, 0x91, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91, 0x49, + 0xf7, 0xbd, 0x97, 0x2a, 0x4a, 0x93, 0xe5, 0x0c, 0xd2, 0x86, 0xc6, 0x81, 0x37, 0x10, 0x8f, 0x53, + 0x1e, 0x24, 0xe9, 0xc0, 0xa9, 0xa0, 0x75, 0x91, 0x45, 0x56, 0x61, 0xe1, 0x91, 0xdf, 0xdf, 0xf5, + 0x02, 0xa7, 0xde, 0xb6, 0x3a, 0x36, 0xd3, 0x94, 0xe1, 0xf3, 0x91, 0x03, 0x39, 0x9f, 0x8f, 0xb2, + 0x74, 0x1b, 0x93, 0xe9, 0x3e, 0x0c, 0xf7, 0x13, 0x1e, 0xf4, 0x79, 0xd4, 0x7f, 0xea, 0x89, 0x17, + 0xce, 0xa2, 0x4a, 0x77, 0x92, 0x2b, 0x6d, 0xb7, 0x78, 0x2c, 0x9c, 0x26, 0x7a, 0xc4, 0x6f, 0xb2, + 0x06, 0xb5, 0x2d, 0x2f, 0xe9, 0x8a, 0x61, 0x72, 0xe4, 0x2c, 0xb5, 0xad, 0x4e, 0x99, 0x65, 0x34, + 0x59, 0x81, 0xca, 0xbe, 0xcb, 0x7d, 0xe1, 0x5c, 0x40, 0x03, 0x45, 0x10, 0x0a, 0x8b, 0xf7, 0xc2, + 0x48, 0x78, 0xcf, 0x02, 0x6c, 0x82, 0xd3, 0xc2, 0xa4, 0x26, 0x78, 0xe4, 0x3d, 0xb0, 0x65, 0x4a, + 0xcb, 0x6d, 0xab, 0xd3, 0xb8, 0xb5, 0xbc, 0x61, 0xfa, 0xb8, 0xd1, 0x15, 0xae, 0x37, 0xe0, 0x3e, + 0x93, 0x52, 0x54, 0xe2, 0x23, 0x87, 0x9c, 0xad, 0xc4, 0x47, 0x94, 0xc2, 0x52, 0x6f, 0x30, 0x0c, + 0xa3, 0x84, 0x89, 0x78, 0x18, 0x06, 0xb1, 0x20, 0x2d, 0xb0, 0x77, 0xa2, 0xc8, 0xb1, 0x30, 0xac, + 0xfc, 0xa4, 0xdf, 0x40, 0x6b, 0xcb, 0x0f, 0xdd, 0xe3, 0x2e, 0x4f, 0x38, 0x13, 0xcf, 0x53, 0x11, + 0x27, 0x12, 0xbb, 0x82, 0xa7, 0xf4, 0x14, 0x21, 0xb9, 0xd8, 0x6f, 0xa7, 0xa4, 0xb8, 0x48, 0xc8, + 0xba, 0x60, 0xd5, 0x54, 0x7b, 0xf0, 0x1b, 0x73, 0x3f, 0xe2, 0x51, 0x1f, 0x7b, 0x5a, 0x66, 0x8a, + 0x90, 0x5c, 0x8c, 0x84, 0x73, 0x50, 0x66, 0x8a, 0xa0, 0x3d, 0x58, 0x2e, 0xc4, 0xd7, 0x30, 0x57, + 0x61, 0x81, 0x85, 0x2f, 0x7a, 0xdd, 0xd8, 0xb1, 0xda, 0x76, 0xa7, 0xcc, 0x34, 0x85, 0x03, 0x13, + 0xfa, 0xe9, 0x20, 0x90, 0xa2, 0x12, 0x8a, 0x72, 0x06, 0xbd, 0x02, 0x15, 0x9c, 0x1e, 0x99, 0x65, + 0x6e, 0x2b, 0x3f, 0xe9, 0xb7, 0x16, 0xd4, 0x77, 0xf9, 0x08, 0x81, 0xc4, 0xe4, 0x0e, 0xd4, 0x4c, + 0x6f, 0x51, 0xa9, 0x71, 0xeb, 0xdd, 0xbc, 0x82, 0x99, 0xda, 0x86, 0xd1, 0xd9, 0x09, 0x92, 0x68, + 0xcc, 0x32, 0x93, 0xb5, 0xcf, 0xa1, 0x39, 0x21, 0x92, 0xf1, 0x8e, 0xc5, 0xd8, 0x54, 0xf5, 0x58, + 0x8c, 0x65, 0xae, 0x27, 0xdc, 0x4f, 0x05, 0xd6, 0xaa, 0xcc, 0x14, 0xf1, 0x59, 0xe9, 0x53, 0x8b, + 0x3e, 0x05, 0xb2, 0x1d, 0x09, 0x9e, 0x08, 0x0c, 0xb2, 0x2b, 0xe2, 0x98, 0x3f, 0x13, 0xf3, 0x2a, + 0x6e, 0x17, 0x2b, 0x9e, 0x55, 0xb7, 0x54, 0xa8, 0x2e, 0xbd, 0x0e, 0xa4, 0x2b, 0x7c, 0x91, 0x08, + 0x7d, 0xba, 0x5f, 0xe3, 0x97, 0x3e, 0x37, 0x18, 0xe6, 0xeb, 0x92, 0x6b, 0x50, 0x96, 0xab, 0x02, + 0x83, 0x35, 0x6e, 0x5d, 0xcc, 0xeb, 0x94, 0x6d, 0x11, 0x86, 0x0a, 0xd8, 0x1b, 0x74, 0xda, 0xdf, + 0x4c, 0x10, 0xb0, 0xcd, 0x72, 0x06, 0xfd, 0xde, 0x32, 0x31, 0x31, 0x89, 0x73, 0xe6, 0x3d, 0x31, + 0x69, 0xd7, 0x35, 0x12, 0x1b, 0x91, 0xac, 0xe6, 0x48, 0x8a, 0x5b, 0x68, 0x16, 0x98, 0xf2, 0x34, + 0x98, 0xbb, 0xa6, 0x56, 0x6f, 0x8a, 0x85, 0xba, 0xf0, 0x7f, 0xe5, 0x61, 0xf3, 0x84, 0x7b, 0x3e, + 0x3f, 0xf4, 0xff, 0x55, 0x3b, 0x27, 0xd2, 0x72, 0xa0, 0x8a, 0xb6, 0xbd, 0xae, 0x3e, 0x18, 0x86, + 0xa4, 0x5f, 0x43, 0x7e, 0xc6, 0x1e, 0xf2, 0x81, 0xd0, 0xde, 0xf0, 0x3b, 0xab, 0x46, 0xe9, 0x1c, + 0xd5, 0x58, 0x81, 0x8a, 0x3c, 0x97, 0x72, 0xcf, 0xdb, 0x32, 0x30, 0x12, 0x73, 0x6a, 0x74, 0x1b, + 0x16, 0xf6, 0xdd, 0x23, 0x31, 0xe0, 0xe4, 0x43, 0xa8, 0x22, 0x7e, 0x11, 0xeb, 0xc3, 0x72, 0x61, + 0x6a, 0x08, 0x98, 0x91, 0xd3, 0x1f, 0x2d, 0x9d, 0xf8, 0x4c, 0xc8, 0x13, 0x01, 0x4b, 0x53, 0x01, + 0xc9, 0x0d, 0xa8, 0x6a, 0xd4, 0xb8, 0x4b, 0xce, 0x98, 0x35, 0xa3, 0x43, 0xae, 0xc1, 0x02, 0x66, + 0x1a, 0x3b, 0xe5, 0x69, 0x50, 0xc8, 0x67, 0x5a, 0x4c, 0x77, 0xc0, 0x7e, 0xc2, 0x7a, 0x72, 0xa5, + 0x60, 0x3e, 0x06, 0x92, 0xa6, 0x24, 0xd0, 0x2f, 0xc2, 0x38, 0xd1, 0x3d, 0xc1, 0x6f, 0xc9, 0xdb, + 0x0b, 0x23, 0x35, 0xc5, 0x4d, 0x86, 0xdf, 0xf4, 0x67, 0x0b, 0xca, 0x0f, 0xc3, 0xbe, 0x20, 0x4b, + 0x50, 0xea, 0x75, 0xb5, 0x93, 0x52, 0xaf, 0x4b, 0xde, 0x41, 0xff, 0xba, 0x0f, 0xcd, 0x1c, 0xc5, + 0x13, 0xd6, 0x63, 0x18, 0xf9, 0x2a, 0xd4, 0x7b, 0xf1, 0x5e, 0xe4, 0x0d, 0x78, 0x34, 0xd6, 0x37, + 0x6d, 0xce, 0xc0, 0xd3, 0x9c, 0xf0, 0x44, 0xdd, 0x7f, 0x75, 0xa6, 0x08, 0x72, 0x0d, 0xaa, 0xf7, + 0xd9, 0xde, 0xb6, 0x74, 0x5c, 0x99, 0xe5, 0xd8, 0x48, 0xe9, 0x5d, 0x68, 0x49, 0x54, 0x68, 0x65, + 0xa6, 0x6f, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xa9, 0xa9, 0x3c, 0x54, 0xa9, 0x10, 0x8a, 0x3e, 0x50, + 0x1e, 0x76, 0x4e, 0x44, 0x90, 0x14, 0xe6, 0x17, 0x69, 0x74, 0xd0, 0x64, 0x8a, 0x20, 0x54, 0x55, + 0x40, 0xa7, 0xba, 0x94, 0x23, 0x92, 0x5c, 0x86, 0x32, 0xfa, 0x83, 0x05, 0x60, 0x00, 0xa5, 0x71, + 0x66, 0x62, 0x9d, 0x6d, 0x42, 0x3a, 0x66, 0xd2, 0xf4, 0xc9, 0x6e, 0xe5, 0x5a, 0x8a, 0xcf, 0xcc, + 0x24, 0x7e, 0x94, 0x4f, 0xa2, 0x6a, 0xfa, 0xa5, 0xa9, 0x11, 0x51, 0x51, 0xf3, 0x79, 0x0c, 0xa0, + 0x51, 0xe0, 0xcf, 0x1c, 0xca, 0x1b, 0xd9, 0x1c, 0x95, 0xa6, 0x5d, 0x22, 0x5f, 0xbb, 0xd4, 0x4a, + 0x73, 0xb6, 0x9c, 0x07, 0x8d, 0x82, 0xd1, 0xcc, 0x78, 0x1d, 0xb8, 0x30, 0xb9, 0x33, 0xcc, 0x45, + 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x27, 0x0b, 0x9a, 0xdb, 0x7e, 0x1a, 0x27, 0x22, 0xd2, 0xd1, 0xa4, + 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xde, 0x87, 0x8a, 0xec, 0x81, 0xda, 0x0c, + 0xa7, 0x1b, 0xa4, 0x84, 0x85, 0x0e, 0x95, 0x5f, 0xdf, 0x21, 0xfa, 0x14, 0x6a, 0x5b, 0xfb, 0xbd, + 0xfb, 0x51, 0x98, 0x0e, 0x67, 0x66, 0x6f, 0xde, 0x88, 0xa5, 0xc2, 0x1b, 0xb1, 0xa5, 0xde, 0x3b, + 0x2a, 0x43, 0x7c, 0xdc, 0xb4, 0xd4, 0xe3, 0xa6, 0xac, 0x39, 0x7c, 0x44, 0xf7, 0x61, 0x59, 0xa5, + 0x2e, 0x57, 0xd7, 0x9b, 0x6c, 0x59, 0xf3, 0x4c, 0xb1, 0xf3, 0x67, 0x8a, 0x74, 0xaa, 0x96, 0xf8, + 0x7f, 0xe9, 0xf4, 0xef, 0x12, 0x2c, 0x33, 0x11, 0x7b, 0x2f, 0x45, 0x2f, 0x88, 0x93, 0x28, 0x75, + 0xe5, 0xba, 0x92, 0xf6, 0x5f, 0x86, 0x87, 0xba, 0x2f, 0x36, 0x53, 0xc4, 0x79, 0x0e, 0x14, 0xe9, + 0x40, 0xb5, 0xb8, 0x3b, 0x4e, 0xab, 0x19, 0x31, 0xb9, 0x09, 0xd5, 0xfd, 0x30, 0x8d, 0xdc, 0xec, + 0x74, 0x14, 0x2e, 0x05, 0x85, 0x48, 0x89, 0x99, 0x51, 0x23, 0x8f, 0x81, 0x1c, 0x44, 0x3c, 0x88, + 0x7d, 0x2e, 0x41, 0x1a, 0xe3, 0xda, 0xf4, 0x8b, 0xa8, 0xa0, 0x33, 0xe1, 0x67, 0x86, 0x31, 0xf9, + 0xb8, 0x78, 0xfc, 0x9d, 0x2a, 0x22, 0x5e, 0x99, 0x44, 0xac, 0x4f, 0x54, 0x71, 0x4d, 0xdc, 0x99, + 0x9a, 0x65, 0x67, 0x01, 0x0d, 0x2f, 0xe7, 0x86, 0x13, 0x62, 0x36, 0xa9, 0x4d, 0xbf, 0xb3, 0x60, + 0xb1, 0x88, 0xec, 0x5c, 0x6b, 0x27, 0x6b, 0x74, 0x69, 0xfe, 0x93, 0xcb, 0x34, 0xba, 0x3c, 0xeb, + 0x91, 0x5b, 0x29, 0x3e, 0xc3, 0x52, 0xb8, 0x7c, 0x46, 0xb9, 0xde, 0x02, 0x54, 0x1b, 0x1a, 0x7b, + 0x3c, 0x4a, 0x3c, 0xe9, 0x52, 0x3f, 0x13, 0x2a, 0xac, 0xc8, 0xa2, 0xc7, 0x70, 0xe5, 0xd4, 0xd0, + 0x6d, 0x87, 0x83, 0xa1, 0x9c, 0xee, 0xb7, 0x18, 0x3e, 0x79, 0x0f, 0x44, 0x51, 0x18, 0x99, 0x6a, + 0x20, 0x41, 0xb7, 0xa0, 0x76, 0x10, 0x0e, 0x43, 0x3f, 0x7c, 0x36, 0x9e, 0xb3, 0x74, 0x1c, 0xa8, + 0xaa, 0xbb, 0x47, 0x2d, 0xb9, 0x3a, 0x33, 0x24, 0xbd, 0x28, 0x4f, 0x89, 0xcb, 0x7d, 0x37, 0xf5, + 0x79, 0x22, 0xf0, 0xd9, 0x8e, 0xcc, 0x07, 0x21, 0xef, 0xab, 0x5d, 0xa2, 0x0f, 0x24, 0x15, 0x7a, + 0x48, 0x39, 0x26, 0x55, 0xb8, 0xe3, 0x36, 0x91, 0x61, 0xee, 0x38, 0x45, 0x91, 0x4f, 0xa0, 0x51, + 0xd0, 0xd6, 0xc9, 0x5d, 0x9a, 0x9a, 0x65, 0x25, 0x64, 0x45, 0x4d, 0xfa, 0x9b, 0x35, 0x61, 0x79, + 0xea, 0x9a, 0xd7, 0x01, 0x4f, 0x54, 0xc1, 0x6a, 0x4c, 0x53, 0xb2, 0x00, 0x3b, 0x23, 0xd7, 0x4f, + 0x63, 0x29, 0xd2, 0xb7, 0x7b, 0xc6, 0x90, 0x05, 0x90, 0x3f, 0x58, 0xc3, 0xd4, 0xbc, 0xb0, 0x0c, + 0x29, 0x7f, 0x3b, 0x76, 0x05, 0xef, 0xfb, 0x5e, 0x20, 0x70, 0x82, 0x6c, 0x96, 0xd1, 0xe4, 0xa6, + 0xda, 0xd5, 0xe6, 0x18, 0xac, 0xcd, 0x84, 0x8f, 0x1a, 0x6a, 0x8f, 0xc7, 0x94, 0x40, 0x6b, 0x5a, + 0x44, 0x57, 0x80, 0xa8, 0x99, 0xd8, 0x3c, 0x0c, 0x23, 0x73, 0xb5, 0xd3, 0x6d, 0xb3, 0x9e, 0x64, + 0x27, 0xe6, 0xbd, 0x18, 0xf2, 0x2a, 0x97, 0x8a, 0x55, 0xde, 0x6a, 0xfd, 0xfe, 0x6a, 0xdd, 0xfa, + 0xe3, 0xd5, 0xba, 0xf5, 0xe7, 0xab, 0x75, 0xeb, 0x97, 0xbf, 0xd6, 0xff, 0x77, 0xb8, 0x80, 0xff, + 0x2e, 0xdc, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0x17, 0x40, 0x19, 0xfb, 0x86, 0x10, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -4379,6 +4419,33 @@ func (m *RecalculateCaches) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *LoadSchemaMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *LoadSchemaMessage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *LoadSchemaMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + return len(dAtA) - i, nil +} + func (m *TransactionMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5432,6 +5499,18 @@ func (m *RecalculateCaches) Size() (n int) { return n } +func (m *LoadSchemaMessage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *TransactionMessage) Size() (n int) { if m == nil { return 0 @@ -10683,6 +10762,60 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { } return nil } +func (m *LoadSchemaMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: LoadSchemaMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: LoadSchemaMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *TransactionMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/internal/private.proto b/internal/private.proto index 7a29abbe2..8cc195fdf 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -208,6 +208,8 @@ message Topology { message RecalculateCaches {} +message LoadSchemaMessage {} + message TransactionMessage { string Action = 1; Transaction Transaction = 2; diff --git a/server.go b/server.go index 216444a0e..93a546218 100644 --- a/server.go +++ b/server.go @@ -854,6 +854,9 @@ func (s *Server) receiveMessage(m Message) error { case *RecalculateCaches: s.holder.recalculateCaches() + case *LoadSchemaMessage: + s.holder.LoadSchema() + case *NodeStatus: s.handleRemoteStatus(obj) From f3d1572232d8767844af10bd6b7f43beaa1fb5e4 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 10 Feb 2021 11:55:34 -0600 Subject: [PATCH 4/4] update comment --- etcd/embed.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/etcd/embed.go b/etcd/embed.go index 0d2d919a4..b50f74a1d 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -680,10 +680,7 @@ func (e *Etcd) View(ctx context.Context, indexName, fieldName, name string) ([]b // CreateView differs from CreateIndex and CreateField in that it does not // return an error if the view already exists. If this logic needs to be -// changed, we likely need to introduce an ErrViewExists variable and return -// that. I decided not to do that now because it would require importing the -// etcd package into the pilosa root package. The better way to do that may be -// to define those error types in the disco package instead. +// changed, we likely need to return disco.ErrViewExists. func (e *Etcd) CreateView(ctx context.Context, indexName, fieldName, name string, val []byte) error { cli, err := e.client() if err != nil {