From 3d270f45d2df9e0affe48044ca99427ab4bceb4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 27 May 2020 21:56:11 +0200 Subject: [PATCH 1/6] Add (in memory) ETag to index and fields --- api.go | 38 +++- api_test.go | 25 ++- cluster.go | 11 +- encoding/proto/proto.go | 12 ++ field.go | 10 + gossip/gossip.go | 10 +- holder.go | 51 +++++- http/handler.go | 272 ++++++++++++++++++---------- index.go | 23 ++- internal/private.pb.go | 392 +++++++++++++++++++++++++++++++--------- internal/private.proto | 6 + pilosa.go | 5 + server.go | 13 +- server/handler_test.go | 60 ++++++ 14 files changed, 706 insertions(+), 222 deletions(-) diff --git a/api.go b/api.go index 210569f6a..43dba98f0 100644 --- a/api.go +++ b/api.go @@ -181,10 +181,15 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + index.mu.Lock() + index.etag = newETag() + index.mu.Unlock() + // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ Index: indexName, + ETag: index.ETag(), Meta: &options, }) if err != nil { @@ -269,14 +274,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + field.mu.Lock() + field.etag = newETag() + field.mu.Unlock() // Send the create field message to all nodes. - err = api.server.SendSync( - &CreateFieldMessage{ - Index: indexName, - Field: fieldName, - Meta: &fo, - }) + err = api.server.SendSync(&CreateFieldMessage{ + Index: indexName, + Field: fieldName, + ETag: field.ETag(), + Meta: &fo, + }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) return nil, errors.Wrap(err, "sending CreateField message") @@ -803,6 +811,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } + // set etags for indexes and fields (if empty), and then apply schema. + for _, index := range s.Indexes { + if index.ETag == 0 { + index.ETag = newETag() + } + for _, field := range index.Fields { + if field.ETag == 0 { + field.ETag = newETag() + } + } + } + if !remote { nodes := api.cluster.Nodes() for i, node := range nodes { @@ -813,7 +833,11 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - return api.holder.applySchema(s) + if err := api.holder.applySchema(s); err != nil { + return errors.Wrap(err, "applying schema") + } + + return nil } // Views returns the views in the given field. diff --git a/api_test.go b/api_test.go index b7f0fc06f..c11ac4f8a 100644 --- a/api_test.go +++ b/api_test.go @@ -186,17 +186,24 @@ func TestAPI_Import(t *testing.T) { t.Run("RowIDColumnKey", func(t *testing.T) { ctx := context.Background() - index := "rick" - field := "f" + indexName := "rick" + fieldName := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if index.ETag() == 0 { + t.Fatal("index etag is empty") + } + + field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } + if field.ETag() == 0 { + t.Fatal("field etag is empty") + } rowID := uint64(1) timestamp := int64(0) @@ -215,8 +222,8 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: index, - Field: field, + Index: indexName, + Field: fieldName, Shard: 0, RowIDs: rowIDs, ColumnKeys: colKeys, @@ -226,10 +233,10 @@ func TestAPI_Import(t *testing.T) { t.Fatal(err) } - pql := fmt.Sprintf("Row(%s=%d)", field, rowID) + pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { t.Fatal(err) } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %#v", keys) @@ -237,7 +244,7 @@ func TestAPI_Import(t *testing.T) { // Query node1. if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { return err } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { return fmt.Errorf("unexpected column keys: %#v", keys) diff --git a/cluster.go b/cluster.go index e979710d8..72ce09b17 100644 --- a/cluster.go +++ b/cluster.go @@ -1272,9 +1272,7 @@ func (c *cluster) listenForJoins() { // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. // We use a bool `setNormal` to indicate when at least one node has joined. var setNormal bool - for { - // Handle all pending joins before changing state back to NORMAL. select { case nodeAction := <-c.joiningLeavingNodes: @@ -2146,7 +2144,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name} + is := &IndexStatus{Name: idx.Name, ETag: idx.ETag} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2155,6 +2153,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, + ETag: f.ETag, AvailableShards: availableShards, }) } @@ -2491,7 +2490,7 @@ type translationResizeNode struct { // Schema contains information about indexes and their configuration. type Schema struct { - Indexes []*IndexInfo + Indexes []*IndexInfo `json:"indexes"` } func encodeTopology(topology *Topology) *internal.Topology { @@ -2530,6 +2529,7 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { Index string + ETag int64 Meta *IndexOptions } @@ -2542,6 +2542,7 @@ type DeleteIndexMessage struct { type CreateFieldMessage struct { Index string Field string + ETag int64 Meta *FieldOptions } @@ -2606,12 +2607,14 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { Name string + ETag int64 Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string + ETag int64 AvailableShards *roaring.Bitmap } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index a4508f652..6eb389302 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -594,6 +594,7 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ Name: idx.Name, + ETag: idx.ETag, Options: s.encodeIndexMeta(&idx.Options), Fields: s.encodeFieldInfos(idx.Fields), } @@ -610,6 +611,7 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ Name: f.Name, + ETag: f.ETag, Meta: s.encodeFieldOptions(&f.Options), Views: make([]string, 0, len(f.Views)), } @@ -686,6 +688,7 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ Index: m.Index, + ETag: m.ETag, Meta: s.encodeIndexMeta(m.Meta), } } @@ -707,6 +710,7 @@ func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *inte return &internal.CreateFieldMessage{ Index: m.Index, Field: m.Field, + ETag: m.ETag, Meta: s.encodeFieldOptions(m.Meta), } } @@ -787,6 +791,7 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ Name: m.Name, + ETag: m.ETag, Fields: s.encodeFieldStatuses(m.Fields), } } @@ -802,6 +807,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, + ETag: m.ETag, AvailableShards: m.AvailableShards.Slice(), } } @@ -938,6 +944,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name + m.ETag = idx.ETag m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -953,6 +960,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name + m.ETag = f.ETag m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -1016,6 +1024,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index + m.ETag = pb.ETag m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1034,6 +1043,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field + m.ETag = pb.ETag m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1107,6 +1117,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name + m.ETag = pb.ETag m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1121,6 +1132,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name + m.ETag = pb.ETag m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } diff --git a/field.go b/field.go index d38aa2ffb..272717565 100644 --- a/field.go +++ b/field.go @@ -87,6 +87,7 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { mu sync.RWMutex + etag int64 path string index string name string @@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } +// ETag is an identifier for a specific version of field. +func (f *Field) ETag() int64 { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.etag +} + // Index returns the index name the field was initialized with. func (f *Field) Index() string { return f.index } @@ -1972,6 +1981,7 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { Name string `json:"name"` + ETag int64 `json:"etag,omitempty"` Options FieldOptions `json:"options"` Views []*ViewInfo `json:"views,omitempty"` } diff --git a/gossip/gossip.go b/gossip/gossip.go index 84f914975..115c4274b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name} + is := &pilosa.IndexStatus{Name: idx.Name, ETag: idx.ETag} + for _, f := range idx.Fields { availableShards := roaring.NewBitmap() if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() } - is.Fields = append(is.Fields, &pilosa.FieldStatus{ + + fs := &pilosa.FieldStatus{ Name: f.Name, + ETag: f.ETag, AvailableShards: availableShards, - }) + } + is.Fields = append(is.Fields, fs) } m.Indexes = append(m.Indexes, is) } diff --git a/holder.go b/holder.go index ad52ee082..7434a0d0c 100644 --- a/holder.go +++ b/holder.go @@ -234,7 +234,13 @@ func (h *Holder) Open() error { return errors.Wrap(err, "opening index") } - if err := index.Open(); err != nil { + if h.isCoordinator() { + index.etag = newETag() + err = index.OpenWithETag() + } else { + err = index.Open() + } + if err != nil { if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue @@ -381,10 +387,15 @@ func (h *Holder) Schema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + ETag: index.ETag(), Options: index.Options(), } for _, field := range index.Fields() { - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + ETag: field.ETag(), + Options: field.Options(), + } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } @@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + ETag: index.ETag(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + ETag: field.ETag(), + Options: field.Options(), + } di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo { // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. - for _, index := range schema.Indexes { - idx, err := h.CreateIndexIfNotExists(index.Name, index.Options) + for _, i := range schema.Indexes { + idx, err := h.CreateIndexIfNotExists(i.Name, i.Options) if err != nil { return errors.Wrap(err, "creating index") } + if i.ETag != 0 { + idx.mu.Lock() + idx.etag = i.ETag + idx.mu.Unlock() + } + // Create fields that don't exist. - for _, f := range index.Fields { - field, err := idx.createFieldIfNotExists(f.Name, &f.Options) + for _, f := range i.Fields { + fld, err := idx.createFieldIfNotExists(f.Name, &f.Options) if err != nil { return errors.Wrap(err, "creating field") } + if f.ETag != 0 { + fld.mu.Lock() + fld.etag = f.ETag + fld.mu.Unlock() + } + // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v.Name) + _, err := fld.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -652,6 +680,13 @@ func (h *Holder) recalculateCaches() { } } +func (h *Holder) isCoordinator() bool { + if s, ok := h.broadcaster.(*Server); ok { + return s.isCoordinator + } + return false +} + // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { oldLimit := &syscall.Rlimit{} diff --git a/http/handler.go b/http/handler.go index 5b9eaf405..555c789c9 100644 --- a/http/handler.go +++ b/http/handler.go @@ -15,13 +15,13 @@ package http import ( + "bytes" "context" "crypto/tls" "encoding/json" "expvar" "fmt" "io" - "io/ioutil" "math" "net" "net/http" @@ -504,7 +504,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -787,8 +787,14 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) - + index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { + w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + } else if _, ok = err.(pilosa.ConflictError); ok { + if index, _ = h.api.Index(r.Context(), indexName); index != nil { + w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + } + } resp.write(w, err) } @@ -925,11 +931,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex)) } - _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) if _, ok := err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } + if field != nil { + w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + } else if _, ok = err.(pilosa.ConflictError); ok { + if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { + w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + } + } resp.write(w, err) } @@ -1240,7 +1253,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error // readProtobufQueryRequest parses query parameters in protobuf from r. func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { // Slurp the body. - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1258,7 +1271,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er q := r.URL.Query() // Parse query string. - buf, err := ioutil.ReadAll(r.Body) + buf, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1319,95 +1332,38 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse return json.NewEncoder(w).Encode(resp) } -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. +func validateProtobufHeader(r *http.Request) (error string, code int) { if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return + return "Unsupported media type", http.StatusUnsupportedMediaType } - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // If the clear flag is true, treat the import as clear bits. - q := r.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + if r.Header.Get("Accept") != "application/x-protobuf" { + return "Not acceptable", http.StatusNotAcceptable } + return +} - // Get index and field type to determine how to handle the - // import data. - field, err := h.api.Field(r.Context(), indexName, fieldName) - if err != nil { - switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: - fallthrough - case pilosa.ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return +func validateETagHeader(r *http.Request, index *pilosa.Index, field *pilosa.Field) (error string, code int) { + etags := strings.Split(r.Header.Get("If-Match"), ",") + netags := len(etags) + for i := 0; i < netags && i < 2; i++ { + etags[i] = strings.TrimLeft(strings.TrimSpace(etags[i]), "W/") } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if netags == 1 && etags[0] != "" && etags[0] != "*" { + return "Precondition Failed", http.StatusPreconditionFailed } - - // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { - // Field type: Int - // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if netags > 1 { + indexETag := etags[0] + if indexETag != "" && strconv.FormatInt(index.ETag(), 10) != indexETag { + return "Precondition Failed", http.StatusPreconditionFailed } - if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.Import(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return + fieldETag := etags[1] + if fieldETag != "" && strconv.FormatInt(field.ETag(), 10) != fieldETag { + return "Precondition Failed", http.StatusPreconditionFailed } } - // Write response. - _, err = w.Write(importOk) - if err != nil { - h.logger.Printf("writing import response: %v", err) - } + return } // handleGetExport handles /export requests. @@ -1885,6 +1841,101 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // Verify if request matches etag + if error, code := validateETagHeader(r, index, field); error != "" { + http.Error(w, error, code) + return + } + + // If the clear flag is true, treat the import as clear bits. + q := r.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + + opts := []pilosa.ImportOption{ + pilosa.OptImportOptionsClear(doClear), + pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + } + + // Read entire body. + body, err := readBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Unmarshal request based on field type. + if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { + // Field type: Int + // Marshal into request object. + req := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + req := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.Import(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + // Write response. + _, err = w.Write(importOk) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } +} + // handlePostImportColumnAttrs func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. @@ -1898,7 +1949,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req opts := []pilosa.ImportOption{} - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1922,18 +1973,38 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } } -// handlPostRoaringImport +// handlePostImportRoaring func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) return } + + // Get index and field type to determine how to handle the + // import data. indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // Verify if request matches etag + if error, code := validateETagHeader(r, index, field); error != "" { + http.Error(w, error, code) + return + } q := r.URL.Query() remoteStr := q.Get("remote") @@ -1946,7 +2017,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Read entire body. span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() if err != nil { @@ -2040,3 +2111,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) h.logger.Printf("writing translate keys response: %v", err) } } + +// Read entire request body. +func readBody(r *http.Request) ([]byte, error) { + var contentLength int64 = bytes.MinRead + if r.ContentLength > 0 { + contentLength = r.ContentLength + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength)) + if _, err := buf.ReadFrom(r.Body); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/index.go b/index.go index a5f143bfc..aaf61e70f 100644 --- a/index.go +++ b/index.go @@ -37,6 +37,7 @@ import ( // Index represents a container for fields. type Index struct { mu sync.RWMutex + etag int64 path string name string keys bool // use string keys @@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } +// ETag is an identifier for a specific version of an index. +func (i *Index) ETag() int64 { + i.mu.RLock() + defer i.mu.RUnlock() + return i.etag +} + // Name returns name of the index. func (i *Index) Name() string { return i.name } @@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open() (err error) { +func (i *Index) Open() error { return i.open(false) } + +// OpenWithETag opens and initializes the index and set a new ETag for fields. +func (i *Index) OpenWithETag() error { return i.open(true) } + +func (i *Index) open(withETag bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -154,7 +167,7 @@ func (i *Index) Open() (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(); err != nil { + if err := i.openFields(withETag); err != nil { return errors.Wrap(err, "opening fields") } @@ -197,7 +210,7 @@ func (i *Index) Open() (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields() error { +func (i *Index) openFields(withETag bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -229,6 +242,9 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if withETag { + fld.etag = newETag() + } mu.Unlock() if err != nil { return errors.Wrapf(ErrName, "'%s'", fi.Name()) @@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` + ETag int64 `json:"etag,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index 38eb7af3d..694935fb0 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } +func (m *CreateIndexMessage) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` + ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } +func (m *CreateFieldMessage) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -853,6 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` + ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -912,6 +929,13 @@ func (m *Field) GetViews() []string { return nil } +func (m *Field) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type Schema struct { Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + ETag int64 `protobuf:"varint,2,opt,name=ETag,proto3" json:"ETag,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1008,6 +1033,13 @@ func (m *Index) GetName() string { return "" } +func (m *Index) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + func (m *Index) GetOptions() *IndexMeta { if m != nil { return m.Options @@ -1340,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1392,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } +func (m *IndexStatus) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` + ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1447,6 +1488,13 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } +func (m *FieldStatus) GetETag() int64 { + if m != nil { + return m.ETag + } + return 0 +} + type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -2421,96 +2469,98 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1418 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0x1b, 0xc5, - 0x13, 0xff, 0xaf, 0x56, 0xb6, 0xa4, 0x96, 0xe5, 0xc8, 0x93, 0xc4, 0xd9, 0xf8, 0x4f, 0x19, 0x31, - 0xa4, 0x88, 0x48, 0x15, 0x26, 0x95, 0x50, 0xc5, 0x67, 0xaa, 0x12, 0x5b, 0x4e, 0x10, 0xc1, 0x8e, - 0x33, 0x72, 0x72, 0xe3, 0x30, 0x5e, 0x4d, 0xc5, 0x5b, 0x5e, 0xed, 0x8a, 0xdd, 0x59, 0x47, 0xce, - 0x81, 0x2b, 0x54, 0xf1, 0x02, 0x1c, 0x38, 0xf0, 0x1e, 0xbc, 0x00, 0x47, 0x1e, 0x81, 0x0a, 0x4f, - 0xc1, 0x8d, 0x9a, 0x9e, 0x99, 0xdd, 0x95, 0x2c, 0xe3, 0x90, 0x70, 0xdb, 0xfe, 0xf5, 0x77, 0x4f, - 0x77, 0xcf, 0x2c, 0xb4, 0xc6, 0x49, 0x70, 0xcc, 0xa5, 0xd8, 0x18, 0x27, 0xb1, 0x8c, 0x49, 0x3d, - 0x88, 0xa4, 0x48, 0x22, 0x1e, 0xae, 0x2d, 0x8d, 0xb3, 0x83, 0x30, 0xf0, 0x35, 0x4e, 0x1f, 0x40, - 0xa3, 0x1f, 0x0d, 0xc5, 0x64, 0x47, 0x48, 0x4e, 0x08, 0x54, 0x1f, 0x8a, 0x93, 0xd4, 0x73, 0x3b, - 0x4e, 0xb7, 0xce, 0xf0, 0x9b, 0xbc, 0x07, 0xcb, 0xfb, 0x09, 0xf7, 0x8f, 0xb6, 0x27, 0x41, 0x2a, - 0x45, 0xe4, 0x0b, 0xaf, 0x8a, 0xdc, 0x19, 0x94, 0xfe, 0xe2, 0xc2, 0xd2, 0xfd, 0x40, 0x84, 0xc3, - 0x47, 0x63, 0x19, 0xc4, 0x51, 0xaa, 0x8c, 0xed, 0x9f, 0x8c, 0x85, 0x57, 0xef, 0x38, 0xdd, 0x06, - 0xc3, 0x6f, 0xf2, 0x16, 0x34, 0xb6, 0xb8, 0x7f, 0x28, 0x90, 0xe1, 0x22, 0xa3, 0x00, 0x72, 0xee, - 0x20, 0x78, 0xa1, 0xbd, 0xb4, 0x58, 0x01, 0x90, 0x0e, 0x34, 0xf7, 0x83, 0x91, 0x78, 0x9c, 0xf1, - 0x48, 0x66, 0x23, 0x6f, 0x01, 0xb5, 0xcb, 0x10, 0x59, 0x85, 0xc5, 0x47, 0xe1, 0x70, 0x27, 0x88, - 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, 0x50, 0x16, 0xe7, 0x13, 0x0f, 0x0a, 0x9c, 0x4f, 0xf2, 0x74, - 0x9b, 0xd3, 0xe9, 0xee, 0xc6, 0x03, 0xc9, 0xa3, 0x21, 0x4f, 0x86, 0x4f, 0x03, 0xf1, 0xdc, 0x5b, - 0xd2, 0xe9, 0x4e, 0xa3, 0x4a, 0x77, 0x93, 0xa7, 0xc2, 0x6b, 0xa1, 0x45, 0xfc, 0x26, 0x6b, 0x50, - 0xdf, 0x0c, 0x64, 0x4f, 0x8c, 0xe5, 0xa1, 0xb7, 0xdc, 0x71, 0xba, 0x55, 0x96, 0xd3, 0xe4, 0x12, - 0x2c, 0x0c, 0x7c, 0x1e, 0x0a, 0xef, 0x02, 0x2a, 0x68, 0x82, 0x50, 0x58, 0xba, 0x1f, 0x27, 0x22, - 0x78, 0x16, 0xe1, 0x21, 0x78, 0x6d, 0x4c, 0x6a, 0x0a, 0x23, 0xef, 0x82, 0xab, 0x52, 0x5a, 0xe9, - 0x38, 0xdd, 0xe6, 0xad, 0x95, 0x0d, 0x7b, 0x8e, 0x1b, 0x3d, 0xe1, 0x07, 0x23, 0x1e, 0x32, 0xc5, - 0x45, 0x21, 0x3e, 0xf1, 0xc8, 0xd9, 0x42, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x38, 0x91, - 0x4c, 0xa4, 0xe3, 0x38, 0x4a, 0x05, 0x69, 0x83, 0xbb, 0x9d, 0x24, 0x9e, 0x83, 0x6e, 0xd5, 0x27, - 0xfd, 0x0e, 0xda, 0x9b, 0x61, 0xec, 0x1f, 0xf5, 0xb8, 0xe4, 0x4c, 0x7c, 0x9b, 0x89, 0x54, 0xaa, - 0xd8, 0x75, 0x78, 0x5a, 0x4e, 0x13, 0x0a, 0xc5, 0xf3, 0xf6, 0x2a, 0x1a, 0x45, 0x42, 0xd5, 0x05, - 0xab, 0xa6, 0x8f, 0x07, 0xbf, 0x31, 0xf7, 0x43, 0x9e, 0x0c, 0xf1, 0x4c, 0xab, 0x4c, 0x13, 0x0a, - 0x45, 0x4f, 0xd8, 0x07, 0x55, 0xa6, 0x09, 0xda, 0x87, 0x95, 0x92, 0x7f, 0x13, 0xe6, 0x2a, 0x2c, - 0xb2, 0xf8, 0x79, 0xbf, 0x97, 0x7a, 0x4e, 0xc7, 0xed, 0x56, 0x99, 0xa1, 0xb0, 0x61, 0xe2, 0x30, - 0x1b, 0x45, 0x8a, 0x55, 0x41, 0x56, 0x01, 0xd0, 0xab, 0xb0, 0x80, 0xdd, 0xa3, 0xb2, 0x2c, 0x74, - 0xd5, 0x27, 0xfd, 0xde, 0x81, 0xc6, 0x0e, 0x9f, 0x60, 0x20, 0x29, 0xb9, 0x03, 0x75, 0x7b, 0xb6, - 0x28, 0xd4, 0xbc, 0xf5, 0x4e, 0x51, 0xc1, 0x5c, 0x6c, 0xc3, 0xca, 0x6c, 0x47, 0x32, 0x39, 0x61, - 0xb9, 0xca, 0xda, 0xe7, 0xd0, 0x9a, 0x62, 0x29, 0x7f, 0x47, 0xe2, 0xc4, 0x56, 0xf5, 0x48, 0x9c, - 0xa8, 0x5c, 0x8f, 0x79, 0x98, 0x09, 0xac, 0x55, 0x95, 0x69, 0xe2, 0xb3, 0xca, 0x27, 0x0e, 0x7d, - 0x0a, 0x64, 0x2b, 0x11, 0x5c, 0x0a, 0x74, 0xb2, 0x23, 0xd2, 0x94, 0x3f, 0x13, 0xe7, 0x55, 0xdc, - 0x2d, 0x57, 0x3c, 0xaf, 0x6e, 0xa5, 0x54, 0x5d, 0x7a, 0x03, 0x48, 0x4f, 0x84, 0x42, 0x0a, 0x33, - 0xdd, 0xff, 0x60, 0x97, 0x0e, 0x6c, 0x0c, 0xe7, 0xcb, 0x92, 0xeb, 0x50, 0x55, 0xab, 0x02, 0x9d, - 0x35, 0x6f, 0x5d, 0x2c, 0xea, 0x94, 0x6f, 0x11, 0x86, 0x02, 0x34, 0xb4, 0x46, 0x31, 0xca, 0x57, - 0x4c, 0x6c, 0xaa, 0x95, 0x6e, 0x18, 0x57, 0x2e, 0xba, 0x5a, 0x2d, 0x5c, 0x95, 0xd7, 0x8c, 0xf1, - 0x76, 0xd7, 0xa6, 0xfb, 0xba, 0xde, 0xa8, 0x0f, 0xff, 0xd7, 0x16, 0xee, 0x1d, 0xf3, 0x20, 0xe4, - 0x07, 0xe1, 0xbf, 0x3a, 0x91, 0xa9, 0xc0, 0x3d, 0xa8, 0xa1, 0x6e, 0xbf, 0x67, 0x7a, 0xdb, 0x92, - 0xf4, 0x1b, 0x28, 0xc6, 0x64, 0x97, 0x8f, 0x84, 0xb1, 0x86, 0xdf, 0x79, 0xbe, 0x95, 0xf3, 0xf3, - 0x55, 0x8e, 0xd5, 0x68, 0xa9, 0x55, 0xed, 0x2a, 0xc7, 0x48, 0xd0, 0xdb, 0xb0, 0x38, 0xf0, 0x0f, - 0xc5, 0x88, 0x93, 0xf7, 0xa1, 0x86, 0x11, 0x8a, 0xd4, 0x74, 0xf4, 0x85, 0x99, 0x93, 0x62, 0x96, - 0x4f, 0x53, 0x93, 0xd9, 0xdc, 0x98, 0x3e, 0x80, 0x9a, 0x71, 0x8c, 0x13, 0x7d, 0xc6, 0x89, 0x5b, - 0x19, 0x72, 0x1d, 0x16, 0x31, 0xd8, 0xd4, 0xab, 0xce, 0x7a, 0x45, 0x9c, 0x19, 0x36, 0xdd, 0x06, - 0xf7, 0x09, 0xeb, 0xab, 0xc1, 0xc6, 0x80, 0xad, 0x53, 0x43, 0xa9, 0x50, 0xbe, 0x8c, 0x53, 0x69, - 0xca, 0x8a, 0xdf, 0x0a, 0xdb, 0x8b, 0x13, 0x89, 0x25, 0x6d, 0x31, 0xfc, 0xa6, 0x3f, 0x3b, 0x50, - 0xdd, 0x8d, 0x87, 0x82, 0x2c, 0x43, 0xa5, 0xdf, 0x33, 0x46, 0x2a, 0xfd, 0x1e, 0x79, 0x1b, 0xed, - 0x9b, 0x52, 0xb6, 0x8a, 0x28, 0x9e, 0xb0, 0x3e, 0x43, 0xcf, 0xd7, 0xa0, 0xd5, 0x4f, 0xb7, 0xe2, - 0x38, 0x19, 0x06, 0x11, 0x97, 0x71, 0x62, 0xee, 0xbc, 0x69, 0x10, 0x67, 0x4b, 0x72, 0xa9, 0x6f, - 0xa3, 0x06, 0xd3, 0x04, 0xb9, 0x0e, 0xb5, 0x07, 0x6c, 0x6f, 0x4b, 0x39, 0x58, 0x98, 0xe7, 0xc0, - 0x72, 0xe9, 0x5d, 0x68, 0xab, 0xe8, 0x50, 0xcb, 0x36, 0xd2, 0x2a, 0x2c, 0x2a, 0x2c, 0x8f, 0xd6, - 0x50, 0x85, 0xab, 0x4a, 0xc9, 0x15, 0xfd, 0x5a, 0x5b, 0xd8, 0x3e, 0x16, 0x91, 0x2c, 0xb5, 0x22, - 0xd2, 0x68, 0xa0, 0xc5, 0x34, 0x41, 0xa8, 0xae, 0x84, 0x49, 0x79, 0xb9, 0x88, 0x48, 0xa1, 0x0c, - 0x79, 0xf4, 0x47, 0x07, 0xc0, 0x06, 0x94, 0xa5, 0xb9, 0x8a, 0x73, 0xb6, 0x0a, 0xe9, 0xda, 0x96, - 0x32, 0x63, 0xd8, 0x2e, 0xa4, 0x34, 0xce, 0x6c, 0xcb, 0x7d, 0x58, 0xb4, 0x9c, 0x3e, 0xfc, 0xcb, - 0x33, 0xad, 0xa2, 0xbd, 0x16, 0x8d, 0xb7, 0x07, 0xcd, 0x12, 0x7e, 0x46, 0xfb, 0xd9, 0x7e, 0xaa, - 0xcc, 0x9a, 0x44, 0xdc, 0x98, 0xb4, 0x5d, 0xf5, 0x10, 0x9a, 0x25, 0x78, 0xae, 0xc5, 0x2e, 0x5c, - 0x98, 0x1e, 0x70, 0x7b, 0x71, 0xcc, 0xc2, 0x34, 0x80, 0xd6, 0x56, 0x98, 0xa5, 0x52, 0x24, 0xc6, - 0x9c, 0xba, 0x6d, 0x34, 0x90, 0x1f, 0x5e, 0x01, 0xcc, 0x3f, 0x3f, 0x72, 0x0d, 0x16, 0x54, 0x19, - 0xf5, 0x9c, 0x9e, 0xae, 0xb1, 0x66, 0xd2, 0xa7, 0x50, 0xdf, 0x1c, 0xf4, 0x1f, 0x24, 0x71, 0x36, - 0x9e, 0x1b, 0xb4, 0x7d, 0x4a, 0x55, 0x4a, 0x4f, 0xa9, 0xb6, 0x7e, 0x16, 0xb8, 0xf8, 0x9c, 0xc0, - 0x37, 0x40, 0x5b, 0xbf, 0x01, 0xaa, 0x06, 0xe1, 0x6a, 0xb1, 0xaf, 0xe8, 0x1d, 0xac, 0xd6, 0xc3, - 0xeb, 0x6c, 0x32, 0x7b, 0x9b, 0xbb, 0xc5, 0x6d, 0xae, 0x8c, 0xea, 0x45, 0xf9, 0x5f, 0x1a, 0xfd, - 0xab, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x0b, 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x27, 0x4a, - 0xff, 0xab, 0xf8, 0xc0, 0x54, 0xdb, 0x65, 0x9a, 0x78, 0x95, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, - 0xdc, 0xa7, 0x45, 0xcb, 0x22, 0xe4, 0x26, 0xd4, 0x06, 0x71, 0x96, 0xf8, 0x79, 0xfb, 0x96, 0x16, - 0xb0, 0x8e, 0x4c, 0xb3, 0x99, 0x15, 0x23, 0x8f, 0x81, 0xec, 0x27, 0x3c, 0x4a, 0x43, 0xae, 0x82, - 0xb5, 0xca, 0xf5, 0xd9, 0x07, 0x44, 0x49, 0x66, 0xca, 0xce, 0x1c, 0x65, 0xf2, 0x51, 0x79, 0x3e, - 0xbd, 0x1a, 0x46, 0x7d, 0x69, 0x3a, 0x6a, 0xd3, 0xf2, 0xe5, 0x39, 0xbe, 0x33, 0xd3, 0xa9, 0xde, - 0x22, 0x2a, 0x5e, 0x29, 0x14, 0xa7, 0xd8, 0x6c, 0x5a, 0x9a, 0xfe, 0xe0, 0xc0, 0x52, 0x39, 0xb2, - 0x57, 0xda, 0x0b, 0xf9, 0x81, 0x57, 0xce, 0x7f, 0xa1, 0xd8, 0x03, 0xaf, 0xce, 0x7b, 0x13, 0x2e, - 0x94, 0x5f, 0x2d, 0x19, 0x5c, 0x39, 0xa3, 0x5c, 0x6f, 0x10, 0x54, 0x07, 0x9a, 0x7b, 0x3c, 0x91, - 0x81, 0x32, 0x69, 0xae, 0xe4, 0x05, 0x56, 0x86, 0xe8, 0x11, 0x5c, 0x3d, 0xd5, 0x7c, 0x5b, 0xf1, - 0x68, 0xac, 0xba, 0xfc, 0x0d, 0x9a, 0x50, 0x2d, 0xea, 0x24, 0x31, 0xed, 0xd7, 0x60, 0x9a, 0xa0, - 0x9f, 0xc2, 0xe5, 0x81, 0x90, 0xa5, 0xd6, 0xb3, 0x33, 0xd4, 0x01, 0x77, 0x57, 0x3c, 0x3f, 0x23, - 0x41, 0xc5, 0xa2, 0x5f, 0x80, 0xf7, 0x64, 0x3c, 0xe4, 0x52, 0xbc, 0x96, 0xf6, 0x26, 0xd4, 0xf7, - 0xe3, 0x71, 0x1c, 0xc6, 0xcf, 0x4e, 0xce, 0xd9, 0x65, 0x1e, 0xd4, 0xf4, 0xad, 0xa4, 0x97, 0x63, - 0x83, 0x59, 0x92, 0x5e, 0x54, 0x63, 0xea, 0xf3, 0xd0, 0xcf, 0x42, 0x15, 0x86, 0x7a, 0x5e, 0xa7, - 0x54, 0x98, 0x41, 0xe0, 0x58, 0xb8, 0xd2, 0x45, 0x77, 0x0f, 0x01, 0x7b, 0xd1, 0x69, 0x8a, 0x7c, - 0x0c, 0xcd, 0x92, 0xb4, 0x29, 0xe0, 0xe5, 0x99, 0x79, 0xd1, 0x4c, 0x56, 0x96, 0xa4, 0xbf, 0x3a, - 0x53, 0x9a, 0xa7, 0xee, 0x7c, 0xe3, 0xf0, 0x58, 0x1f, 0x4a, 0x9d, 0x19, 0x4a, 0xe5, 0xba, 0x3d, - 0xf1, 0xc3, 0x2c, 0x55, 0x2c, 0x7d, 0xcd, 0x17, 0x80, 0xca, 0x55, 0xfd, 0x43, 0xc6, 0x99, 0x34, - 0x9b, 0xd3, 0x92, 0xea, 0x77, 0xae, 0x27, 0xf8, 0x30, 0x0c, 0x22, 0x81, 0x5d, 0xea, 0xb2, 0x9c, - 0x26, 0x37, 0xf5, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x37, 0x7c, 0x94, 0xd0, 0x37, 0x41, 0x4a, 0x09, - 0xb4, 0x67, 0x59, 0x9b, 0xed, 0xdf, 0x5e, 0xae, 0x3b, 0xbf, 0xbf, 0x5c, 0x77, 0xfe, 0x78, 0xb9, - 0xee, 0xfc, 0xf4, 0xe7, 0xfa, 0xff, 0x0e, 0x16, 0xf1, 0xaf, 0xfc, 0xf6, 0xdf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x8a, 0x80, 0x2a, 0x36, 0xbe, 0x0f, 0x00, 0x00, + // 1448 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xc5, + 0x17, 0xff, 0xaf, 0xd7, 0x4e, 0xec, 0xe3, 0x38, 0x75, 0xa6, 0x6d, 0xba, 0xcd, 0x1f, 0x05, 0x33, + 0x54, 0xd4, 0x54, 0x6a, 0xa8, 0x5a, 0x24, 0x3e, 0x2b, 0xb5, 0x89, 0xdd, 0x62, 0x20, 0x69, 0x3b, + 0x4e, 0x7b, 0x8b, 0x26, 0xf6, 0x28, 0x59, 0x65, 0xbd, 0xeb, 0xee, 0xce, 0xa6, 0x4e, 0x2f, 0x10, + 0x77, 0x20, 0xf1, 0x02, 0x5c, 0x70, 0xc1, 0x7b, 0xf0, 0x02, 0x5c, 0xf2, 0x08, 0xa8, 0x3c, 0x05, + 0x77, 0x68, 0xce, 0xcc, 0xec, 0xae, 0x1d, 0x87, 0x84, 0x96, 0xbb, 0x39, 0x5f, 0x73, 0x7e, 0xe7, + 0x73, 0xc7, 0x86, 0xc6, 0x38, 0xf6, 0x8f, 0xb8, 0x14, 0x1b, 0xe3, 0x38, 0x92, 0x11, 0xa9, 0xfa, + 0xa1, 0x14, 0x71, 0xc8, 0x83, 0xb5, 0xa5, 0x71, 0xba, 0x17, 0xf8, 0x03, 0xcd, 0xa7, 0x0f, 0xa1, + 0xd6, 0x0b, 0x87, 0x62, 0xb2, 0x2d, 0x24, 0x27, 0x04, 0xca, 0x5f, 0x89, 0xe3, 0xc4, 0x73, 0x5b, + 0x4e, 0xbb, 0xca, 0xf0, 0x4c, 0xde, 0x83, 0xe5, 0xdd, 0x98, 0x0f, 0x0e, 0xbb, 0x13, 0x3f, 0x91, + 0x22, 0x1c, 0x08, 0xaf, 0x8c, 0xd2, 0x19, 0x2e, 0xfd, 0xc5, 0x85, 0xa5, 0x07, 0xbe, 0x08, 0x86, + 0x8f, 0xc6, 0xd2, 0x8f, 0xc2, 0x44, 0x5d, 0xb6, 0x7b, 0x3c, 0x16, 0x5e, 0xb5, 0xe5, 0xb4, 0x6b, + 0x0c, 0xcf, 0xe4, 0x2d, 0xa8, 0x6d, 0xf1, 0xc1, 0x81, 0x40, 0x81, 0x8b, 0x82, 0x9c, 0x91, 0x49, + 0xfb, 0xfe, 0x4b, 0xed, 0xa5, 0xc1, 0x72, 0x06, 0x69, 0x41, 0x7d, 0xd7, 0x1f, 0x89, 0x27, 0x29, + 0x0f, 0x65, 0x3a, 0xf2, 0x2a, 0x68, 0x5d, 0x64, 0x91, 0x55, 0x58, 0x78, 0x14, 0x0c, 0xb7, 0xfd, + 0xd0, 0xab, 0xb5, 0x9c, 0xb6, 0xcb, 0x0c, 0x65, 0xf9, 0x7c, 0xe2, 0x41, 0xce, 0xe7, 0x93, 0x2c, + 0xdc, 0xfa, 0x74, 0xb8, 0x3b, 0x51, 0x5f, 0xf2, 0x70, 0xc8, 0xe3, 0xe1, 0x33, 0x5f, 0xbc, 0xf0, + 0x96, 0x74, 0xb8, 0xd3, 0x5c, 0x65, 0xbb, 0xc9, 0x13, 0xe1, 0x35, 0xf0, 0x46, 0x3c, 0x93, 0x35, + 0xa8, 0x6e, 0xfa, 0xb2, 0x23, 0xc6, 0xf2, 0xc0, 0x5b, 0x6e, 0x39, 0xed, 0x32, 0xcb, 0x68, 0x72, + 0x09, 0x2a, 0xfd, 0x01, 0x0f, 0x84, 0x77, 0x01, 0x0d, 0x34, 0x41, 0x28, 0x2c, 0x3d, 0x88, 0x62, + 0xe1, 0xef, 0x87, 0x58, 0x04, 0xaf, 0x89, 0x41, 0x4d, 0xf1, 0xc8, 0xbb, 0xe0, 0xaa, 0x90, 0x56, + 0x5a, 0x4e, 0xbb, 0x7e, 0x7b, 0x65, 0xc3, 0xd6, 0x71, 0xa3, 0x23, 0x06, 0xfe, 0x88, 0x07, 0x4c, + 0x49, 0x51, 0x89, 0x4f, 0x3c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xbd, 0xd1, 0x38, 0x8a, + 0x25, 0x13, 0xc9, 0x38, 0x0a, 0x13, 0x41, 0x9a, 0xe0, 0x76, 0xe3, 0xd8, 0x73, 0xd0, 0xad, 0x3a, + 0xd2, 0x6f, 0xa1, 0xb9, 0x19, 0x44, 0x83, 0xc3, 0x0e, 0x97, 0x9c, 0x89, 0xe7, 0xa9, 0x48, 0xa4, + 0xc2, 0xae, 0xe1, 0x69, 0x3d, 0x4d, 0x28, 0x2e, 0xd6, 0xdb, 0x2b, 0x69, 0x2e, 0x12, 0x2a, 0x2f, + 0x98, 0x35, 0x5d, 0x1e, 0x3c, 0x63, 0xec, 0x07, 0x3c, 0x1e, 0x62, 0x4d, 0xcb, 0x4c, 0x13, 0x8a, + 0x8b, 0x9e, 0xb0, 0x0f, 0xca, 0x4c, 0x13, 0xb4, 0x07, 0x2b, 0x05, 0xff, 0x06, 0xe6, 0x2a, 0x2c, + 0xb0, 0xe8, 0x45, 0xaf, 0x93, 0x78, 0x4e, 0xcb, 0x6d, 0x97, 0x99, 0xa1, 0xb0, 0x61, 0xa2, 0x20, + 0x1d, 0x85, 0x4a, 0x54, 0x42, 0x51, 0xce, 0xa0, 0x57, 0xa1, 0x82, 0xdd, 0xa3, 0xa2, 0xcc, 0x6d, + 0xd5, 0x91, 0x7e, 0xef, 0x40, 0x6d, 0x9b, 0x4f, 0x10, 0x48, 0x42, 0xee, 0x42, 0xd5, 0xd6, 0x16, + 0x95, 0xea, 0xb7, 0xdf, 0xc9, 0x33, 0x98, 0xa9, 0x6d, 0x58, 0x9d, 0x6e, 0x28, 0xe3, 0x63, 0x96, + 0x99, 0xac, 0x7d, 0x06, 0x8d, 0x29, 0x91, 0xf2, 0x77, 0x28, 0x8e, 0x6d, 0x56, 0x0f, 0xc5, 0xb1, + 0x8a, 0xf5, 0x88, 0x07, 0xa9, 0xc0, 0x5c, 0x95, 0x99, 0x26, 0x3e, 0x2d, 0x7d, 0xec, 0xd0, 0x67, + 0x40, 0xb6, 0x62, 0xc1, 0xa5, 0x40, 0x27, 0xdb, 0x22, 0x49, 0xf8, 0xbe, 0x38, 0x2b, 0xe3, 0x6e, + 0x31, 0xe3, 0x59, 0x76, 0x4b, 0x85, 0xec, 0xd2, 0x1b, 0x40, 0x3a, 0x22, 0x10, 0x52, 0x98, 0xe9, + 0xfe, 0x87, 0x7b, 0xe9, 0xbe, 0xc5, 0x70, 0xb6, 0x2e, 0xb9, 0x0e, 0x65, 0xb5, 0x2a, 0xd0, 0x59, + 0xfd, 0xf6, 0xc5, 0x3c, 0x4f, 0xd9, 0x16, 0x61, 0x65, 0xbb, 0x4b, 0xba, 0xbb, 0x7c, 0x1f, 0xb1, + 0xba, 0x0c, 0xcf, 0xf4, 0x3b, 0xc7, 0x7a, 0x42, 0xe8, 0xe7, 0x8c, 0x76, 0xaa, 0xbf, 0x6e, 0x18, + 0xff, 0x2e, 0xfa, 0x5f, 0xcd, 0xfd, 0x17, 0x77, 0xcf, 0x0c, 0x84, 0x72, 0x01, 0xc2, 0x3d, 0x9b, + 0x97, 0xd7, 0x45, 0x40, 0x07, 0xf0, 0x7f, 0x7d, 0xc3, 0xfd, 0x23, 0xee, 0x07, 0x7c, 0x2f, 0xf8, + 0x57, 0xa5, 0x9b, 0x0a, 0xc6, 0x83, 0x45, 0xb4, 0xed, 0x75, 0xcc, 0x10, 0x58, 0x92, 0x3e, 0x87, + 0x7c, 0x9e, 0x76, 0xf8, 0x48, 0x98, 0xdb, 0xf0, 0x9c, 0xe5, 0xa0, 0x74, 0x8e, 0x1c, 0x5c, 0x82, + 0x8a, 0x9a, 0x41, 0xb5, 0xd3, 0x5d, 0xe5, 0x18, 0x89, 0xb9, 0x99, 0xb9, 0x03, 0x0b, 0xfd, 0xc1, + 0x81, 0x18, 0x71, 0xf2, 0x3e, 0x2c, 0x22, 0x6a, 0x91, 0x98, 0x71, 0xb8, 0x30, 0x53, 0x66, 0x66, + 0xe5, 0x6a, 0x90, 0x4c, 0x84, 0xf3, 0x80, 0x5a, 0x37, 0xa5, 0xdc, 0x0d, 0xb9, 0x09, 0x8b, 0x06, + 0x21, 0xee, 0x88, 0x53, 0x7a, 0xc8, 0xea, 0x90, 0xeb, 0xb0, 0x80, 0x51, 0x25, 0x5e, 0x79, 0x16, + 0x0a, 0xf2, 0x99, 0x11, 0xd3, 0x2e, 0xb8, 0x4f, 0x59, 0x4f, 0xad, 0x0a, 0x8c, 0xc2, 0x02, 0x31, + 0x94, 0x82, 0xf2, 0x45, 0x94, 0x48, 0x93, 0x7f, 0x3c, 0x2b, 0xde, 0xe3, 0x28, 0x96, 0x98, 0xfb, + 0x06, 0xc3, 0x33, 0xfd, 0xd9, 0x81, 0xf2, 0x4e, 0x34, 0x14, 0x64, 0x19, 0x4a, 0xbd, 0x8e, 0xb9, + 0xa4, 0xd4, 0xeb, 0x90, 0xb7, 0xf1, 0x7e, 0x93, 0xf3, 0x46, 0x8e, 0xe2, 0x29, 0xeb, 0x31, 0xf4, + 0x7c, 0x0d, 0x1a, 0xbd, 0x64, 0x2b, 0x8a, 0xe2, 0xa1, 0x1f, 0x72, 0x19, 0xc5, 0xe6, 0x2b, 0x3a, + 0xcd, 0xc4, 0x69, 0x95, 0x5c, 0xea, 0xef, 0x5b, 0x8d, 0x69, 0x82, 0x5c, 0x87, 0xc5, 0x87, 0xec, + 0xf1, 0x96, 0x72, 0x50, 0x99, 0xe7, 0xc0, 0x4a, 0xe9, 0x3d, 0x68, 0x2a, 0x74, 0x68, 0x65, 0x3b, + 0x6e, 0x15, 0x16, 0x14, 0x2f, 0x43, 0x6b, 0xa8, 0xdc, 0x55, 0xa9, 0xe0, 0x8a, 0x7e, 0xad, 0x6f, + 0xe8, 0x1e, 0x89, 0x50, 0x16, 0x7a, 0x16, 0x69, 0xbc, 0xa0, 0xc1, 0x34, 0x41, 0xa8, 0xce, 0x84, + 0x09, 0x79, 0x39, 0x47, 0xa4, 0xb8, 0x0c, 0x65, 0xf4, 0x47, 0x07, 0xc0, 0x02, 0x4a, 0x93, 0xcc, + 0xc4, 0x39, 0xdd, 0x84, 0xb4, 0x6d, 0x9f, 0x99, 0x19, 0x6e, 0xe6, 0x5a, 0x9a, 0xcf, 0x6c, 0x1f, + 0x7e, 0x90, 0xf7, 0xa1, 0x2e, 0xfe, 0xe5, 0x99, 0x56, 0xd1, 0x5e, 0xf3, 0x6e, 0x1c, 0x42, 0xbd, + 0xc0, 0x9f, 0xdb, 0x92, 0x37, 0xb3, 0x7e, 0x2a, 0xcd, 0x5e, 0x89, 0x7c, 0x73, 0xa5, 0x51, 0x9a, + 0xbb, 0xc5, 0xbe, 0x81, 0x7a, 0x41, 0x75, 0xae, 0x97, 0x36, 0x5c, 0x98, 0xde, 0x0e, 0xf6, 0xf3, + 0x34, 0xcb, 0x9e, 0xeb, 0xc0, 0x87, 0xc6, 0x56, 0x90, 0x26, 0x52, 0xc4, 0xc6, 0x85, 0xfa, 0xce, + 0x69, 0x46, 0x56, 0xe4, 0x9c, 0x31, 0xbf, 0xce, 0xe4, 0x1a, 0x54, 0x54, 0xba, 0xf5, 0xe0, 0x9f, + 0xac, 0x85, 0x16, 0xd2, 0x67, 0x50, 0xdd, 0xec, 0xf7, 0x1e, 0xc6, 0x51, 0x3a, 0x3e, 0x6d, 0x82, + 0xf1, 0xad, 0x56, 0x2a, 0x3c, 0xe2, 0x9a, 0xfa, 0x41, 0xa2, 0x11, 0xe3, 0xeb, 0xa3, 0xa9, 0x5f, + 0x1f, 0x65, 0xc3, 0xe1, 0x13, 0xda, 0x87, 0x15, 0xbd, 0xe8, 0xd5, 0xbe, 0x79, 0x9d, 0xd5, 0x68, + 0xdf, 0x11, 0x6e, 0xfe, 0x8e, 0x50, 0x97, 0xea, 0xcd, 0xfb, 0x5f, 0x5e, 0xfa, 0x57, 0x09, 0x56, + 0x98, 0x48, 0xfc, 0x97, 0xa2, 0x17, 0x26, 0x32, 0x4e, 0x07, 0x6a, 0xef, 0x28, 0xfb, 0x2f, 0xa3, + 0x3d, 0x93, 0x6d, 0x97, 0x69, 0xe2, 0x3c, 0x13, 0x41, 0x6e, 0x41, 0x7d, 0x76, 0x09, 0x9c, 0x54, + 0x2d, 0xaa, 0x90, 0x5b, 0xb0, 0xd8, 0x8f, 0xd2, 0x78, 0x90, 0xb5, 0x79, 0x61, 0xa3, 0x6b, 0x64, + 0x5a, 0xcc, 0xac, 0x1a, 0x79, 0x02, 0x64, 0x37, 0xe6, 0x61, 0x12, 0x70, 0x05, 0xd6, 0x1a, 0x57, + 0x67, 0x9f, 0x2e, 0x05, 0x9d, 0xa9, 0x7b, 0xe6, 0x18, 0x93, 0x0f, 0x8b, 0x73, 0xec, 0x2d, 0x22, + 0xea, 0x4b, 0xd3, 0xa8, 0xcd, 0x68, 0x14, 0xe7, 0xfd, 0xee, 0x4c, 0xa7, 0x7a, 0x0b, 0x68, 0x78, + 0x25, 0x37, 0x9c, 0x12, 0xb3, 0x69, 0x6d, 0xfa, 0x83, 0x03, 0x4b, 0x45, 0x64, 0xe7, 0xda, 0x1f, + 0x59, 0xc1, 0x4b, 0x67, 0xbf, 0x8d, 0x6c, 0xc1, 0xcb, 0xf3, 0x5e, 0xa3, 0x95, 0xe2, 0x7b, 0x29, + 0x85, 0x2b, 0xa7, 0xa4, 0xeb, 0x0d, 0x40, 0xb5, 0xa0, 0xfe, 0x98, 0xc7, 0xd2, 0x57, 0x57, 0x9a, + 0x6f, 0x7c, 0x85, 0x15, 0x59, 0xf4, 0x10, 0xae, 0x9e, 0x68, 0xbe, 0xad, 0x68, 0x34, 0x56, 0x5d, + 0xfe, 0x06, 0x4d, 0xa8, 0x16, 0x7a, 0x1c, 0x9b, 0xf6, 0xab, 0x31, 0x4d, 0xd0, 0x4f, 0xe0, 0x72, + 0x5f, 0xc8, 0x42, 0xeb, 0xd9, 0x19, 0x6a, 0x81, 0xbb, 0x23, 0x5e, 0x9c, 0x12, 0xa0, 0x12, 0xd1, + 0xcf, 0xc1, 0x7b, 0x3a, 0x1e, 0x72, 0x29, 0x5e, 0xcb, 0x7a, 0x13, 0xaa, 0xbb, 0xd1, 0x38, 0x0a, + 0xa2, 0xfd, 0xe3, 0x33, 0x76, 0x99, 0x07, 0x8b, 0xfa, 0xeb, 0xa5, 0x17, 0x66, 0x8d, 0x59, 0x92, + 0x5e, 0x54, 0x63, 0x3a, 0xe0, 0xc1, 0x20, 0x0d, 0x14, 0x0c, 0xf5, 0xb0, 0x4f, 0xa8, 0x30, 0x83, + 0xc0, 0x31, 0x71, 0x85, 0x0f, 0xe2, 0x7d, 0x64, 0xd8, 0x0f, 0xa2, 0xa6, 0xc8, 0x47, 0x50, 0x2f, + 0x68, 0x9b, 0x04, 0x5e, 0x9e, 0x99, 0x17, 0x2d, 0x64, 0x45, 0x4d, 0xfa, 0xab, 0x33, 0x65, 0x79, + 0xe2, 0x6d, 0x60, 0x1c, 0x1e, 0xe9, 0xa2, 0x54, 0x99, 0xa1, 0x54, 0xac, 0xdd, 0xc9, 0x20, 0x48, + 0x13, 0x25, 0xd2, 0xcf, 0x81, 0x9c, 0xa1, 0x62, 0x55, 0xbf, 0x5e, 0xa3, 0x54, 0x9a, 0xcd, 0x69, + 0x49, 0xf5, 0x43, 0xb2, 0x23, 0xf8, 0x30, 0xf0, 0x43, 0x81, 0x5d, 0xea, 0xb2, 0x8c, 0x26, 0xb7, + 0xf4, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x17, 0x3e, 0x6a, 0xe8, 0x2f, 0x41, 0x42, 0x09, 0x34, 0x67, + 0x45, 0x9b, 0xcd, 0xdf, 0x5e, 0xad, 0x3b, 0xbf, 0xbf, 0x5a, 0x77, 0xfe, 0x78, 0xb5, 0xee, 0xfc, + 0xf4, 0xe7, 0xfa, 0xff, 0xf6, 0x16, 0xf0, 0xff, 0x80, 0x3b, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, + 0x4f, 0xdc, 0x5a, 0x3d, 0x38, 0x10, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3043,6 +3093,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3089,6 +3144,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x20 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3229,6 +3289,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x20 + } if len(m.Views) > 0 { for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Views[iNdEx]) @@ -3351,6 +3416,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x10 + } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) @@ -3656,6 +3726,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if len(m.Fields) > 0 { for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { { @@ -3704,6 +3779,11 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ETag != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + i-- + dAtA[i] = 0x18 + } if len(m.AvailableShards) > 0 { dAtA19 := make([]byte, len(m.AvailableShards)*10) var j18 int @@ -4759,6 +4839,9 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4783,6 +4866,9 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4852,6 +4938,9 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4886,6 +4975,9 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -5037,6 +5129,9 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5060,6 +5155,9 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } + if m.ETag != 0 { + n += 1 + sovPrivate(uint64(m.ETag)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7021,6 +7119,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7175,6 +7292,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7584,6 +7720,25 @@ func (m *Field) Unmarshal(dAtA []byte) error { } m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7758,6 +7913,25 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -8682,6 +8856,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8844,6 +9037,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType) } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + } + m.ETag = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ETag |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 849fff04a..502d3b7bd 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,12 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; + int64 ETag = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; + int64 ETag = 4; } message DeleteFieldMessage { @@ -87,6 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; + int64 ETag = 4; } message Schema { @@ -95,6 +98,7 @@ message Schema { message Index { string Name = 1; + int64 ETag = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -132,11 +136,13 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; + int64 ETag = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; + int64 ETag = 3; } message ClusterStatus { diff --git a/pilosa.go b/pilosa.go index d7eb2f3c8..e49af5406 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,6 +17,7 @@ package pilosa import ( "encoding/json" "regexp" + "time" "github.com/pkg/errors" ) @@ -191,6 +192,10 @@ func stringSlicesAreEqual(a, b []string) bool { return true } +func newETag() int64 { + return time.Now().UTC().UnixNano() +} + // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/server.go b/server.go index fd07ec0b2..586d14e73 100644 --- a/server.go +++ b/server.go @@ -356,14 +356,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { } s.executor = newExecutor(executorOpts...) - // s.holder.translateFile.logger = s.logger - path, err := expandDirName(s.dataDir) if err != nil { return nil, err } s.holder.Path = path - // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger s.holder.Stats.SetLogger(s.logger) @@ -716,10 +713,13 @@ func (s *Server) receiveMessage(m Message) error { } case *CreateIndexMessage: opt := obj.Meta - _, err := s.holder.CreateIndex(obj.Index, *opt) + idx, err := s.holder.CreateIndex(obj.Index, *opt) if err != nil { return err } + idx.mu.Lock() + idx.etag = obj.ETag + idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err @@ -730,10 +730,13 @@ func (s *Server) receiveMessage(m Message) error { return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta - _, err := idx.createFieldIfNotExists(obj.Field, opt) + fld, err := idx.createFieldIfNotExists(obj.Field, opt) if err != nil { return err } + fld.mu.Lock() + fld.etag = obj.ETag + fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index b01b15c59..9cd25f50e 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -203,6 +203,53 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("Import", func(t *testing.T) { + indexInfo := cmd.API.Schema(context.Background()) + err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false) + if err != nil { + t.Fatalf("applying schema: %v", err) + } + + idx := indexInfo[0] + fld := indexInfo[0].Fields[0] + msg := pilosa.ImportRequest{ + Index: idx.Name, + Field: fld.Name, + Shard: 0, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) + etag := fmt.Sprintf("%d, %d", idx.ETag, fld.ETag) + + httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", etag) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + if w.Body.String() != "" { + t.Fatalf(w.Body.String()) + } + + etag = "invalid-index-etag, invalid-field-etag" + + httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", etag) + + h.ServeHTTP(w, httpReq) + + if strings.TrimSpace(w.Body.String()) != "Precondition Failed" { + t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) + } + }) + t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") @@ -217,10 +264,23 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatal(err) } + idx, err := cmd.API.Index(context.Background(), "i0") + if err != nil { + t.Fatal(err) + } + fld, err := cmd.API.Field(context.Background(), "i0", "f1") + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("If-Match", fmt.Sprintf("%d, %d", idx.ETag(), fld.ETag())) h.ServeHTTP(w, httpReq) + if w.Body.String() != "" { + t.Fatalf("Unexpected response body: %s", w.Body.String()) + } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) From ba7f039dd1a0369b3b58c407bfd2f5b09cbc7dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Tue, 26 May 2020 00:33:52 +0200 Subject: [PATCH 2/6] Rename etag to createdAt --- api.go | 64 ++++--- api_test.go | 22 +-- cluster.go | 26 +-- encoding/proto/proto.go | 102 ++++++----- field.go | 24 +-- gossip/gossip.go | 4 +- handler.go | 64 +++++-- holder.go | 32 ++-- http/handler.go | 83 +++------ index.go | 32 ++-- internal/private.pb.go | 304 ++++++++++++++++----------------- internal/private.proto | 12 +- internal/public.pb.go | 365 +++++++++++++++++++++++++++++++--------- internal/public.proto | 8 + pilosa.go | 14 +- server.go | 4 +- server/handler_test.go | 183 ++++++++++++++------ 17 files changed, 838 insertions(+), 505 deletions(-) diff --git a/api.go b/api.go index 43dba98f0..0a29ad30a 100644 --- a/api.go +++ b/api.go @@ -182,15 +182,15 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "creating index") } index.mu.Lock() - index.etag = newETag() + index.createdAt = timestamp() index.mu.Unlock() // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ - Index: indexName, - ETag: index.ETag(), - Meta: &options, + Index: indexName, + CreatedAt: index.CreatedAt(), + Meta: &options, }) if err != nil { return nil, errors.Wrap(err, "sending CreateIndex message") @@ -275,15 +275,15 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, errors.Wrap(err, "creating field") } field.mu.Lock() - field.etag = newETag() + field.createdAt = timestamp() field.mu.Unlock() // Send the create field message to all nodes. err = api.server.SendSync(&CreateFieldMessage{ - Index: indexName, - Field: fieldName, - ETag: field.ETag(), - Meta: &fo, + Index: indexName, + Field: fieldName, + CreatedAt: field.CreatedAt(), + Meta: &fo, }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) @@ -421,14 +421,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "validating api method") } - nodes := api.cluster.shardNodes(indexName, shard) - - field := api.holder.Field(indexName, fieldName) - if field == nil { + index, field, err := api.indexField(indexName, fieldName, shard) + if index == nil || field == nil { return newNotFoundError(ErrFieldNotFound) } - errCh := make(chan error, len(nodes)) + if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return newPreconditionFailedError(err) + } + + nodes := api.cluster.shardNodes(indexName, shard) + errCh := make(chan error, len(nodes)) for _, node := range nodes { node := node if node.ID == api.server.nodeID { @@ -811,14 +814,14 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } - // set etags for indexes and fields (if empty), and then apply schema. + // set CreatedAt for indexes and fields (if empty), and then apply schema. for _, index := range s.Indexes { - if index.ETag == 0 { - index.ETag = newETag() + if index.CreatedAt == 0 { + index.CreatedAt = timestamp() } for _, field := range index.Fields { - if field.ETag == 0 { - field.ETag = newETag() + if field.CreatedAt == 0 { + field.CreatedAt = timestamp() } } } @@ -1023,16 +1026,20 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "validating api method") } + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + // Set up import options. options, err := setUpImportOptions(opts...) if err != nil { return errors.Wrap(err, "setting up import options") } - - index, field, err := api.indexField(req.Index, req.Field, req.Shard) - if err != nil { - return errors.Wrap(err, "getting index and field") - } span.LogKV( "index", req.Index, "field", req.Field) @@ -1139,7 +1146,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating api method") } - if err := req.Validate(); err != nil { + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { return errors.Wrap(err, "validating import value request") } @@ -1149,7 +1161,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "setting up import options") } - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + index, field, err = api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } diff --git a/api_test.go b/api_test.go index c11ac4f8a..e8b4f28f5 100644 --- a/api_test.go +++ b/api_test.go @@ -193,16 +193,16 @@ func TestAPI_Import(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - if index.ETag() == 0 { - t.Fatal("index etag is empty") + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") } field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } - if field.ETag() == 0 { - t.Fatal("field etag is empty") + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") } rowID := uint64(1) @@ -222,12 +222,14 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: indexName, - Field: fieldName, - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), + Field: fieldName, + FieldCreatedAt: field.CreatedAt(), + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, } if err := m0.API.Import(ctx, req); err != nil { t.Fatal(err) diff --git a/cluster.go b/cluster.go index 72ce09b17..d283593a1 100644 --- a/cluster.go +++ b/cluster.go @@ -2144,7 +2144,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name, ETag: idx.ETag} + is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2153,7 +2153,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, - ETag: f.ETag, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, }) } @@ -2528,9 +2528,9 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { - Index string - ETag int64 - Meta *IndexOptions + Index string + CreatedAt int64 + Meta *IndexOptions } // DeleteIndexMessage is an internal message indicating index deletion. @@ -2540,10 +2540,10 @@ type DeleteIndexMessage struct { // CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { - Index string - Field string - ETag int64 - Meta *FieldOptions + Index string + Field string + CreatedAt int64 + Meta *FieldOptions } // DeleteFieldMessage is an internal message indicating field deletion. @@ -2606,15 +2606,15 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { - Name string - ETag int64 - Fields []*FieldStatus + Name string + CreatedAt int64 + Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string - ETag int64 + CreatedAt int64 AvailableShards *roaring.Bitmap } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 6eb389302..7d647ded0 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { return &internal.ImportRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - RowIDs: m.RowIDs, - ColumnIDs: m.ColumnIDs, - RowKeys: m.RowKeys, - ColumnKeys: m.ColumnKeys, - Timestamps: m.Timestamps, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, } } func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { return &internal.ImportValueRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - ColumnIDs: m.ColumnIDs, - ColumnKeys: m.ColumnKeys, - Values: m.Values, - FloatValues: m.FloatValues, - StringValues: m.StringValues, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + FloatValues: m.FloatValues, + StringValues: m.StringValues, } } @@ -438,10 +442,12 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, } } @@ -593,10 +599,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ - Name: idx.Name, - ETag: idx.ETag, - Options: s.encodeIndexMeta(&idx.Options), - Fields: s.encodeFieldInfos(idx.Fields), + Name: idx.Name, + CreatedAt: idx.CreatedAt, + Options: s.encodeIndexMeta(&idx.Options), + Fields: s.encodeFieldInfos(idx.Fields), } } @@ -610,10 +616,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ - Name: f.Name, - ETag: f.ETag, - Meta: s.encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), + Name: f.Name, + CreatedAt: f.CreatedAt, + Meta: s.encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), } for _, viewinfo := range f.Views { @@ -687,9 +693,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ - Index: m.Index, - ETag: m.ETag, - Meta: s.encodeIndexMeta(m.Meta), + Index: m.Index, + CreatedAt: m.CreatedAt, + Meta: s.encodeIndexMeta(m.Meta), } } @@ -708,10 +714,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - ETag: m.ETag, - Meta: s.encodeFieldOptions(m.Meta), + Index: m.Index, + Field: m.Field, + CreatedAt: m.CreatedAt, + Meta: s.encodeFieldOptions(m.Meta), } } @@ -790,9 +796,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ - Name: m.Name, - ETag: m.ETag, - Fields: s.encodeFieldStatuses(m.Fields), + Name: m.Name, + CreatedAt: m.CreatedAt, + Fields: s.encodeFieldStatuses(m.Fields), } } @@ -807,7 +813,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, - ETag: m.ETag, + CreatedAt: m.CreatedAt, AvailableShards: m.AvailableShards.Slice(), } } @@ -944,7 +950,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name - m.ETag = idx.ETag + m.CreatedAt = idx.CreatedAt m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -960,7 +966,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name - m.ETag = f.ETag + m.CreatedAt = f.CreatedAt m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -1024,7 +1030,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1043,7 +1049,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1117,7 +1123,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1132,7 +1138,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name - m.ETag = pb.ETag + m.CreatedAt = pb.CreatedAt m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } @@ -1161,6 +1167,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im m.RowKeys = pb.RowKeys m.ColumnKeys = pb.ColumnKeys m.Timestamps = pb.Timestamps + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { @@ -1172,6 +1180,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m m.Values = pb.Values m.FloatValues = pb.FloatValues m.StringValues = pb.StringValues + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { @@ -1183,6 +1193,8 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Action = pb.Action m.Block = int(pb.Block) m.Views = views + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { diff --git a/field.go b/field.go index 272717565..84bdbec73 100644 --- a/field.go +++ b/field.go @@ -86,11 +86,11 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { - mu sync.RWMutex - etag int64 - path string - index string - name string + mu sync.RWMutex + createdAt int64 + path string + index string + name string viewMap map[string]*view @@ -383,12 +383,12 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } -// ETag is an identifier for a specific version of field. -func (f *Field) ETag() int64 { +// CreatedAt is an timestamp for a specific version of field. +func (f *Field) CreatedAt() int64 { f.mu.RLock() defer f.mu.RUnlock() - return f.etag + return f.createdAt } // Index returns the index name the field was initialized with. @@ -1980,10 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - ETag int64 `json:"etag,omitempty"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/gossip/gossip.go b/gossip/gossip.go index 115c4274b..975c57a19 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,7 +324,7 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name, ETag: idx.ETag} + is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { availableShards := roaring.NewBitmap() @@ -334,7 +334,7 @@ func (g *memberSet) LocalState(join bool) []byte { fs := &pilosa.FieldStatus{ Name: f.Name, - ETag: f.ETag, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, } is.Fields = append(is.Fields, fs) diff --git a/handler.go b/handler.go index 3c40d20ad..79c60e285 100644 --- a/handler.go +++ b/handler.go @@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. type ImportValueRequest struct { - Index string - Field string + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 @@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) { // Validate ensures that the payload of the request is valid. func (ivr *ImportValueRequest) Validate() error { + return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt) +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if ivr.Index == "" || ivr.Field == "" { return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field) } @@ -160,6 +167,11 @@ func (ivr *ImportValueRequest) Validate() error { if valueSetCount > 1 { return errors.Errorf("must pass ints, floats, or strings but not multiple") } + if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 { + if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } return nil } @@ -176,14 +188,26 @@ type ImportColumnAttrsRequest struct { // ImportRequest describes the import request structure // for an import. type ImportRequest struct { - Index string - Field string - Shard uint64 - RowIDs []uint64 - ColumnIDs []uint64 - RowKeys []string - ColumnKeys []string - Timestamps []int64 + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 { + if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } const ( @@ -195,10 +219,22 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 { + if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } // ImportResponse is the structured response of an import. diff --git a/holder.go b/holder.go index 7434a0d0c..dd995566a 100644 --- a/holder.go +++ b/holder.go @@ -235,8 +235,8 @@ func (h *Holder) Open() error { } if h.isCoordinator() { - index.etag = newETag() - err = index.OpenWithETag() + index.createdAt = timestamp() + err = index.OpenWithTimestamp() } else { err = index.Open() } @@ -386,15 +386,15 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ - Name: index.Name(), - ETag: index.ETag(), - Options: index.Options(), + Name: index.Name(), + CreatedAt: index.CreatedAt(), + Options: index.Options(), } for _, field := range index.Fields() { fi := &FieldInfo{ - Name: field.Name(), - ETag: field.ETag(), - Options: field.Options(), + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) @@ -415,7 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), - ETag: index.ETag(), + CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -424,9 +424,9 @@ func (h *Holder) limitedSchema() []*IndexInfo { continue } fi := &FieldInfo{ - Name: field.Name(), - ETag: field.ETag(), - Options: field.Options(), + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), } di.Fields = append(di.Fields, fi) } @@ -445,9 +445,9 @@ func (h *Holder) applySchema(schema *Schema) error { if err != nil { return errors.Wrap(err, "creating index") } - if i.ETag != 0 { + if i.CreatedAt != 0 { idx.mu.Lock() - idx.etag = i.ETag + idx.createdAt = i.CreatedAt idx.mu.Unlock() } @@ -457,9 +457,9 @@ func (h *Holder) applySchema(schema *Schema) error { if err != nil { return errors.Wrap(err, "creating field") } - if f.ETag != 0 { + if f.CreatedAt != 0 { fld.mu.Lock() - fld.etag = f.ETag + fld.createdAt = f.CreatedAt fld.mu.Unlock() } diff --git a/http/handler.go b/http/handler.go index 555c789c9..e2787d77c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -401,9 +401,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { - h *Handler - Success bool `json:"success"` - Error *Error `json:"error,omitempty"` + h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *Error `json:"error,omitempty"` } // check determines success or failure based on the error. @@ -773,7 +775,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: indexName} // Decode request. req := postIndexRequest{ @@ -788,11 +790,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { - w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) - } else if _, ok = err.(pilosa.ConflictError); ok { + resp.CreatedAt = index.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { if index, _ = h.api.Index(r.Context(), indexName); index != nil { - w.Header().Add("ETag", strconv.FormatInt(index.ETag(), 10)) + resp.CreatedAt = index.CreatedAt() } } resp.write(w, err) @@ -859,7 +862,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: fieldName} // Decode request. var req postFieldRequest @@ -937,10 +940,10 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } if field != nil { - w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) - } else if _, ok = err.(pilosa.ConflictError); ok { + resp.CreatedAt = field.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { - w.Header().Add("ETag", strconv.FormatInt(field.ETag(), 10)) + resp.CreatedAt = field.CreatedAt() } } resp.write(w, err) @@ -1342,30 +1345,6 @@ func validateProtobufHeader(r *http.Request) (error string, code int) { return } -func validateETagHeader(r *http.Request, index *pilosa.Index, field *pilosa.Field) (error string, code int) { - etags := strings.Split(r.Header.Get("If-Match"), ",") - netags := len(etags) - for i := 0; i < netags && i < 2; i++ { - etags[i] = strings.TrimLeft(strings.TrimSpace(etags[i]), "W/") - } - if netags == 1 && etags[0] != "" && etags[0] != "*" { - return "Precondition Failed", http.StatusPreconditionFailed - } - if netags > 1 { - indexETag := etags[0] - if indexETag != "" && strconv.FormatInt(index.ETag(), 10) != indexETag { - return "Precondition Failed", http.StatusPreconditionFailed - } - - fieldETag := etags[1] - if fieldETag != "" && strconv.FormatInt(field.ETag(), 10) != fieldETag { - return "Precondition Failed", http.StatusPreconditionFailed - } - } - - return -} - // handleGetExport handles /export requests. func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { switch r.Header.Get("Accept") { @@ -1868,12 +1847,6 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Verify if request matches etag - if error, code := validateETagHeader(r, index, field); error != "" { - http.Error(w, error, code) - return - } - // If the clear flag is true, treat the import as clear bits. q := r.URL.Query() doClear := q.Get("clear") == "true" @@ -1902,7 +1875,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1920,7 +1893,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { if err := h.api.Import(r.Context(), req, opts...); err != nil { switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1984,27 +1957,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Get index and field type to determine how to handle the // import data. indexName := mux.Vars(r)["index"] - index, err := h.api.Index(r.Context(), indexName) - if err != nil { - if errors.Cause(err) == pilosa.ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } fieldName := mux.Vars(r)["field"] - field := index.Field(fieldName) - if field == nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - // Verify if request matches etag - if error, code := validateETagHeader(r, index, field); error != "" { - http.Error(w, error, code) - return - } q := r.URL.Query() remoteStr := q.Get("remote") @@ -2047,6 +2000,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) + } else if _, ok := err.(pilosa.NotFoundError); ok { + w.WriteHeader(http.StatusNotFound) + } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) } diff --git a/index.go b/index.go index aaf61e70f..481d22cf4 100644 --- a/index.go +++ b/index.go @@ -36,11 +36,11 @@ import ( // Index represents a container for fields. type Index struct { - mu sync.RWMutex - etag int64 - path string - name string - keys bool // use string keys + mu sync.RWMutex + createdAt int64 + path string + name string + keys bool // use string keys // Existence tracking. trackExistence bool @@ -104,11 +104,11 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } -// ETag is an identifier for a specific version of an index. -func (i *Index) ETag() int64 { +// CreatedAt is an timestamp for a specific version of an index. +func (i *Index) CreatedAt() int64 { i.mu.RLock() defer i.mu.RUnlock() - return i.etag + return i.createdAt } // Name returns name of the index. @@ -150,10 +150,10 @@ func (i *Index) options() IndexOptions { // Open opens and initializes the index. func (i *Index) Open() error { return i.open(false) } -// OpenWithETag opens and initializes the index and set a new ETag for fields. -func (i *Index) OpenWithETag() error { return i.open(true) } +// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. +func (i *Index) OpenWithTimestamp() error { return i.open(true) } -func (i *Index) open(withETag bool) (err error) { +func (i *Index) open(withTimestamp bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -167,7 +167,7 @@ func (i *Index) open(withETag bool) (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(withETag); err != nil { + if err := i.openFields(withTimestamp); err != nil { return errors.Wrap(err, "opening fields") } @@ -210,7 +210,7 @@ func (i *Index) open(withETag bool) (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields(withETag bool) error { +func (i *Index) openFields(withTimestamp bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -242,8 +242,8 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) - if withETag { - fld.etag = newETag() + if withTimestamp { + fld.createdAt = timestamp() } mu.Unlock() if err != nil { @@ -575,7 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - ETag int64 `json:"etag,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index 694935fb0..ea281b409 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,7 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -669,9 +669,9 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } -func (m *CreateIndexMessage) GetETag() int64 { +func (m *CreateIndexMessage) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -680,7 +680,7 @@ type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` - ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -740,9 +740,9 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } -func (m *CreateFieldMessage) GetETag() int64 { +func (m *CreateFieldMessage) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -869,7 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` - ETag int64 `protobuf:"varint,4,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -929,9 +929,9 @@ func (m *Field) GetViews() []string { return nil } -func (m *Field) GetETag() int64 { +func (m *Field) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -985,7 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - ETag int64 `protobuf:"varint,2,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1033,9 +1033,9 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetETag() int64 { +func (m *Index) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -1372,7 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1425,9 +1425,9 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } -func (m *IndexStatus) GetETag() int64 { +func (m *IndexStatus) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -1435,7 +1435,7 @@ func (m *IndexStatus) GetETag() int64 { type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` - ETag int64 `protobuf:"varint,3,opt,name=ETag,proto3" json:"ETag,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1488,9 +1488,9 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } -func (m *FieldStatus) GetETag() int64 { +func (m *FieldStatus) GetCreatedAt() int64 { if m != nil { - return m.ETag + return m.CreatedAt } return 0 } @@ -2469,98 +2469,98 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1448 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xc5, - 0x17, 0xff, 0xaf, 0xd7, 0x4e, 0xec, 0xe3, 0x38, 0x75, 0xa6, 0x6d, 0xba, 0xcd, 0x1f, 0x05, 0x33, - 0x54, 0xd4, 0x54, 0x6a, 0xa8, 0x5a, 0x24, 0x3e, 0x2b, 0xb5, 0x89, 0xdd, 0x62, 0x20, 0x69, 0x3b, - 0x4e, 0x7b, 0x8b, 0x26, 0xf6, 0x28, 0x59, 0x65, 0xbd, 0xeb, 0xee, 0xce, 0xa6, 0x4e, 0x2f, 0x10, - 0x77, 0x20, 0xf1, 0x02, 0x5c, 0x70, 0xc1, 0x7b, 0xf0, 0x02, 0x5c, 0xf2, 0x08, 0xa8, 0x3c, 0x05, - 0x77, 0x68, 0xce, 0xcc, 0xec, 0xae, 0x1d, 0x87, 0x84, 0x96, 0xbb, 0x39, 0x5f, 0x73, 0x7e, 0xe7, - 0x73, 0xc7, 0x86, 0xc6, 0x38, 0xf6, 0x8f, 0xb8, 0x14, 0x1b, 0xe3, 0x38, 0x92, 0x11, 0xa9, 0xfa, - 0xa1, 0x14, 0x71, 0xc8, 0x83, 0xb5, 0xa5, 0x71, 0xba, 0x17, 0xf8, 0x03, 0xcd, 0xa7, 0x0f, 0xa1, - 0xd6, 0x0b, 0x87, 0x62, 0xb2, 0x2d, 0x24, 0x27, 0x04, 0xca, 0x5f, 0x89, 0xe3, 0xc4, 0x73, 0x5b, - 0x4e, 0xbb, 0xca, 0xf0, 0x4c, 0xde, 0x83, 0xe5, 0xdd, 0x98, 0x0f, 0x0e, 0xbb, 0x13, 0x3f, 0x91, - 0x22, 0x1c, 0x08, 0xaf, 0x8c, 0xd2, 0x19, 0x2e, 0xfd, 0xc5, 0x85, 0xa5, 0x07, 0xbe, 0x08, 0x86, - 0x8f, 0xc6, 0xd2, 0x8f, 0xc2, 0x44, 0x5d, 0xb6, 0x7b, 0x3c, 0x16, 0x5e, 0xb5, 0xe5, 0xb4, 0x6b, - 0x0c, 0xcf, 0xe4, 0x2d, 0xa8, 0x6d, 0xf1, 0xc1, 0x81, 0x40, 0x81, 0x8b, 0x82, 0x9c, 0x91, 0x49, - 0xfb, 0xfe, 0x4b, 0xed, 0xa5, 0xc1, 0x72, 0x06, 0x69, 0x41, 0x7d, 0xd7, 0x1f, 0x89, 0x27, 0x29, - 0x0f, 0x65, 0x3a, 0xf2, 0x2a, 0x68, 0x5d, 0x64, 0x91, 0x55, 0x58, 0x78, 0x14, 0x0c, 0xb7, 0xfd, - 0xd0, 0xab, 0xb5, 0x9c, 0xb6, 0xcb, 0x0c, 0x65, 0xf9, 0x7c, 0xe2, 0x41, 0xce, 0xe7, 0x93, 0x2c, - 0xdc, 0xfa, 0x74, 0xb8, 0x3b, 0x51, 0x5f, 0xf2, 0x70, 0xc8, 0xe3, 0xe1, 0x33, 0x5f, 0xbc, 0xf0, - 0x96, 0x74, 0xb8, 0xd3, 0x5c, 0x65, 0xbb, 0xc9, 0x13, 0xe1, 0x35, 0xf0, 0x46, 0x3c, 0x93, 0x35, - 0xa8, 0x6e, 0xfa, 0xb2, 0x23, 0xc6, 0xf2, 0xc0, 0x5b, 0x6e, 0x39, 0xed, 0x32, 0xcb, 0x68, 0x72, - 0x09, 0x2a, 0xfd, 0x01, 0x0f, 0x84, 0x77, 0x01, 0x0d, 0x34, 0x41, 0x28, 0x2c, 0x3d, 0x88, 0x62, - 0xe1, 0xef, 0x87, 0x58, 0x04, 0xaf, 0x89, 0x41, 0x4d, 0xf1, 0xc8, 0xbb, 0xe0, 0xaa, 0x90, 0x56, - 0x5a, 0x4e, 0xbb, 0x7e, 0x7b, 0x65, 0xc3, 0xd6, 0x71, 0xa3, 0x23, 0x06, 0xfe, 0x88, 0x07, 0x4c, - 0x49, 0x51, 0x89, 0x4f, 0x3c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xbd, 0xd1, 0x38, 0x8a, - 0x25, 0x13, 0xc9, 0x38, 0x0a, 0x13, 0x41, 0x9a, 0xe0, 0x76, 0xe3, 0xd8, 0x73, 0xd0, 0xad, 0x3a, - 0xd2, 0x6f, 0xa1, 0xb9, 0x19, 0x44, 0x83, 0xc3, 0x0e, 0x97, 0x9c, 0x89, 0xe7, 0xa9, 0x48, 0xa4, - 0xc2, 0xae, 0xe1, 0x69, 0x3d, 0x4d, 0x28, 0x2e, 0xd6, 0xdb, 0x2b, 0x69, 0x2e, 0x12, 0x2a, 0x2f, - 0x98, 0x35, 0x5d, 0x1e, 0x3c, 0x63, 0xec, 0x07, 0x3c, 0x1e, 0x62, 0x4d, 0xcb, 0x4c, 0x13, 0x8a, - 0x8b, 0x9e, 0xb0, 0x0f, 0xca, 0x4c, 0x13, 0xb4, 0x07, 0x2b, 0x05, 0xff, 0x06, 0xe6, 0x2a, 0x2c, - 0xb0, 0xe8, 0x45, 0xaf, 0x93, 0x78, 0x4e, 0xcb, 0x6d, 0x97, 0x99, 0xa1, 0xb0, 0x61, 0xa2, 0x20, - 0x1d, 0x85, 0x4a, 0x54, 0x42, 0x51, 0xce, 0xa0, 0x57, 0xa1, 0x82, 0xdd, 0xa3, 0xa2, 0xcc, 0x6d, - 0xd5, 0x91, 0x7e, 0xef, 0x40, 0x6d, 0x9b, 0x4f, 0x10, 0x48, 0x42, 0xee, 0x42, 0xd5, 0xd6, 0x16, - 0x95, 0xea, 0xb7, 0xdf, 0xc9, 0x33, 0x98, 0xa9, 0x6d, 0x58, 0x9d, 0x6e, 0x28, 0xe3, 0x63, 0x96, - 0x99, 0xac, 0x7d, 0x06, 0x8d, 0x29, 0x91, 0xf2, 0x77, 0x28, 0x8e, 0x6d, 0x56, 0x0f, 0xc5, 0xb1, - 0x8a, 0xf5, 0x88, 0x07, 0xa9, 0xc0, 0x5c, 0x95, 0x99, 0x26, 0x3e, 0x2d, 0x7d, 0xec, 0xd0, 0x67, - 0x40, 0xb6, 0x62, 0xc1, 0xa5, 0x40, 0x27, 0xdb, 0x22, 0x49, 0xf8, 0xbe, 0x38, 0x2b, 0xe3, 0x6e, - 0x31, 0xe3, 0x59, 0x76, 0x4b, 0x85, 0xec, 0xd2, 0x1b, 0x40, 0x3a, 0x22, 0x10, 0x52, 0x98, 0xe9, - 0xfe, 0x87, 0x7b, 0xe9, 0xbe, 0xc5, 0x70, 0xb6, 0x2e, 0xb9, 0x0e, 0x65, 0xb5, 0x2a, 0xd0, 0x59, - 0xfd, 0xf6, 0xc5, 0x3c, 0x4f, 0xd9, 0x16, 0x61, 0x65, 0xbb, 0x4b, 0xba, 0xbb, 0x7c, 0x1f, 0xb1, - 0xba, 0x0c, 0xcf, 0xf4, 0x3b, 0xc7, 0x7a, 0x42, 0xe8, 0xe7, 0x8c, 0x76, 0xaa, 0xbf, 0x6e, 0x18, - 0xff, 0x2e, 0xfa, 0x5f, 0xcd, 0xfd, 0x17, 0x77, 0xcf, 0x0c, 0x84, 0x72, 0x01, 0xc2, 0x3d, 0x9b, - 0x97, 0xd7, 0x45, 0x40, 0x07, 0xf0, 0x7f, 0x7d, 0xc3, 0xfd, 0x23, 0xee, 0x07, 0x7c, 0x2f, 0xf8, - 0x57, 0xa5, 0x9b, 0x0a, 0xc6, 0x83, 0x45, 0xb4, 0xed, 0x75, 0xcc, 0x10, 0x58, 0x92, 0x3e, 0x87, - 0x7c, 0x9e, 0x76, 0xf8, 0x48, 0x98, 0xdb, 0xf0, 0x9c, 0xe5, 0xa0, 0x74, 0x8e, 0x1c, 0x5c, 0x82, - 0x8a, 0x9a, 0x41, 0xb5, 0xd3, 0x5d, 0xe5, 0x18, 0x89, 0xb9, 0x99, 0xb9, 0x03, 0x0b, 0xfd, 0xc1, - 0x81, 0x18, 0x71, 0xf2, 0x3e, 0x2c, 0x22, 0x6a, 0x91, 0x98, 0x71, 0xb8, 0x30, 0x53, 0x66, 0x66, - 0xe5, 0x6a, 0x90, 0x4c, 0x84, 0xf3, 0x80, 0x5a, 0x37, 0xa5, 0xdc, 0x0d, 0xb9, 0x09, 0x8b, 0x06, - 0x21, 0xee, 0x88, 0x53, 0x7a, 0xc8, 0xea, 0x90, 0xeb, 0xb0, 0x80, 0x51, 0x25, 0x5e, 0x79, 0x16, - 0x0a, 0xf2, 0x99, 0x11, 0xd3, 0x2e, 0xb8, 0x4f, 0x59, 0x4f, 0xad, 0x0a, 0x8c, 0xc2, 0x02, 0x31, - 0x94, 0x82, 0xf2, 0x45, 0x94, 0x48, 0x93, 0x7f, 0x3c, 0x2b, 0xde, 0xe3, 0x28, 0x96, 0x98, 0xfb, - 0x06, 0xc3, 0x33, 0xfd, 0xd9, 0x81, 0xf2, 0x4e, 0x34, 0x14, 0x64, 0x19, 0x4a, 0xbd, 0x8e, 0xb9, - 0xa4, 0xd4, 0xeb, 0x90, 0xb7, 0xf1, 0x7e, 0x93, 0xf3, 0x46, 0x8e, 0xe2, 0x29, 0xeb, 0x31, 0xf4, - 0x7c, 0x0d, 0x1a, 0xbd, 0x64, 0x2b, 0x8a, 0xe2, 0xa1, 0x1f, 0x72, 0x19, 0xc5, 0xe6, 0x2b, 0x3a, - 0xcd, 0xc4, 0x69, 0x95, 0x5c, 0xea, 0xef, 0x5b, 0x8d, 0x69, 0x82, 0x5c, 0x87, 0xc5, 0x87, 0xec, - 0xf1, 0x96, 0x72, 0x50, 0x99, 0xe7, 0xc0, 0x4a, 0xe9, 0x3d, 0x68, 0x2a, 0x74, 0x68, 0x65, 0x3b, - 0x6e, 0x15, 0x16, 0x14, 0x2f, 0x43, 0x6b, 0xa8, 0xdc, 0x55, 0xa9, 0xe0, 0x8a, 0x7e, 0xad, 0x6f, - 0xe8, 0x1e, 0x89, 0x50, 0x16, 0x7a, 0x16, 0x69, 0xbc, 0xa0, 0xc1, 0x34, 0x41, 0xa8, 0xce, 0x84, - 0x09, 0x79, 0x39, 0x47, 0xa4, 0xb8, 0x0c, 0x65, 0xf4, 0x47, 0x07, 0xc0, 0x02, 0x4a, 0x93, 0xcc, - 0xc4, 0x39, 0xdd, 0x84, 0xb4, 0x6d, 0x9f, 0x99, 0x19, 0x6e, 0xe6, 0x5a, 0x9a, 0xcf, 0x6c, 0x1f, - 0x7e, 0x90, 0xf7, 0xa1, 0x2e, 0xfe, 0xe5, 0x99, 0x56, 0xd1, 0x5e, 0xf3, 0x6e, 0x1c, 0x42, 0xbd, - 0xc0, 0x9f, 0xdb, 0x92, 0x37, 0xb3, 0x7e, 0x2a, 0xcd, 0x5e, 0x89, 0x7c, 0x73, 0xa5, 0x51, 0x9a, - 0xbb, 0xc5, 0xbe, 0x81, 0x7a, 0x41, 0x75, 0xae, 0x97, 0x36, 0x5c, 0x98, 0xde, 0x0e, 0xf6, 0xf3, - 0x34, 0xcb, 0x9e, 0xeb, 0xc0, 0x87, 0xc6, 0x56, 0x90, 0x26, 0x52, 0xc4, 0xc6, 0x85, 0xfa, 0xce, - 0x69, 0x46, 0x56, 0xe4, 0x9c, 0x31, 0xbf, 0xce, 0xe4, 0x1a, 0x54, 0x54, 0xba, 0xf5, 0xe0, 0x9f, - 0xac, 0x85, 0x16, 0xd2, 0x67, 0x50, 0xdd, 0xec, 0xf7, 0x1e, 0xc6, 0x51, 0x3a, 0x3e, 0x6d, 0x82, - 0xf1, 0xad, 0x56, 0x2a, 0x3c, 0xe2, 0x9a, 0xfa, 0x41, 0xa2, 0x11, 0xe3, 0xeb, 0xa3, 0xa9, 0x5f, - 0x1f, 0x65, 0xc3, 0xe1, 0x13, 0xda, 0x87, 0x15, 0xbd, 0xe8, 0xd5, 0xbe, 0x79, 0x9d, 0xd5, 0x68, - 0xdf, 0x11, 0x6e, 0xfe, 0x8e, 0x50, 0x97, 0xea, 0xcd, 0xfb, 0x5f, 0x5e, 0xfa, 0x57, 0x09, 0x56, - 0x98, 0x48, 0xfc, 0x97, 0xa2, 0x17, 0x26, 0x32, 0x4e, 0x07, 0x6a, 0xef, 0x28, 0xfb, 0x2f, 0xa3, - 0x3d, 0x93, 0x6d, 0x97, 0x69, 0xe2, 0x3c, 0x13, 0x41, 0x6e, 0x41, 0x7d, 0x76, 0x09, 0x9c, 0x54, - 0x2d, 0xaa, 0x90, 0x5b, 0xb0, 0xd8, 0x8f, 0xd2, 0x78, 0x90, 0xb5, 0x79, 0x61, 0xa3, 0x6b, 0x64, - 0x5a, 0xcc, 0xac, 0x1a, 0x79, 0x02, 0x64, 0x37, 0xe6, 0x61, 0x12, 0x70, 0x05, 0xd6, 0x1a, 0x57, - 0x67, 0x9f, 0x2e, 0x05, 0x9d, 0xa9, 0x7b, 0xe6, 0x18, 0x93, 0x0f, 0x8b, 0x73, 0xec, 0x2d, 0x22, - 0xea, 0x4b, 0xd3, 0xa8, 0xcd, 0x68, 0x14, 0xe7, 0xfd, 0xee, 0x4c, 0xa7, 0x7a, 0x0b, 0x68, 0x78, - 0x25, 0x37, 0x9c, 0x12, 0xb3, 0x69, 0x6d, 0xfa, 0x83, 0x03, 0x4b, 0x45, 0x64, 0xe7, 0xda, 0x1f, - 0x59, 0xc1, 0x4b, 0x67, 0xbf, 0x8d, 0x6c, 0xc1, 0xcb, 0xf3, 0x5e, 0xa3, 0x95, 0xe2, 0x7b, 0x29, - 0x85, 0x2b, 0xa7, 0xa4, 0xeb, 0x0d, 0x40, 0xb5, 0xa0, 0xfe, 0x98, 0xc7, 0xd2, 0x57, 0x57, 0x9a, - 0x6f, 0x7c, 0x85, 0x15, 0x59, 0xf4, 0x10, 0xae, 0x9e, 0x68, 0xbe, 0xad, 0x68, 0x34, 0x56, 0x5d, - 0xfe, 0x06, 0x4d, 0xa8, 0x16, 0x7a, 0x1c, 0x9b, 0xf6, 0xab, 0x31, 0x4d, 0xd0, 0x4f, 0xe0, 0x72, - 0x5f, 0xc8, 0x42, 0xeb, 0xd9, 0x19, 0x6a, 0x81, 0xbb, 0x23, 0x5e, 0x9c, 0x12, 0xa0, 0x12, 0xd1, - 0xcf, 0xc1, 0x7b, 0x3a, 0x1e, 0x72, 0x29, 0x5e, 0xcb, 0x7a, 0x13, 0xaa, 0xbb, 0xd1, 0x38, 0x0a, - 0xa2, 0xfd, 0xe3, 0x33, 0x76, 0x99, 0x07, 0x8b, 0xfa, 0xeb, 0xa5, 0x17, 0x66, 0x8d, 0x59, 0x92, - 0x5e, 0x54, 0x63, 0x3a, 0xe0, 0xc1, 0x20, 0x0d, 0x14, 0x0c, 0xf5, 0xb0, 0x4f, 0xa8, 0x30, 0x83, - 0xc0, 0x31, 0x71, 0x85, 0x0f, 0xe2, 0x7d, 0x64, 0xd8, 0x0f, 0xa2, 0xa6, 0xc8, 0x47, 0x50, 0x2f, - 0x68, 0x9b, 0x04, 0x5e, 0x9e, 0x99, 0x17, 0x2d, 0x64, 0x45, 0x4d, 0xfa, 0xab, 0x33, 0x65, 0x79, - 0xe2, 0x6d, 0x60, 0x1c, 0x1e, 0xe9, 0xa2, 0x54, 0x99, 0xa1, 0x54, 0xac, 0xdd, 0xc9, 0x20, 0x48, - 0x13, 0x25, 0xd2, 0xcf, 0x81, 0x9c, 0xa1, 0x62, 0x55, 0xbf, 0x5e, 0xa3, 0x54, 0x9a, 0xcd, 0x69, - 0x49, 0xf5, 0x43, 0xb2, 0x23, 0xf8, 0x30, 0xf0, 0x43, 0x81, 0x5d, 0xea, 0xb2, 0x8c, 0x26, 0xb7, - 0xf4, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x17, 0x3e, 0x6a, 0xe8, 0x2f, 0x41, 0x42, 0x09, 0x34, 0x67, - 0x45, 0x9b, 0xcd, 0xdf, 0x5e, 0xad, 0x3b, 0xbf, 0xbf, 0x5a, 0x77, 0xfe, 0x78, 0xb5, 0xee, 0xfc, - 0xf4, 0xe7, 0xfa, 0xff, 0xf6, 0x16, 0xf0, 0xff, 0x80, 0x3b, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, - 0x4f, 0xdc, 0x5a, 0x3d, 0x38, 0x10, 0x00, 0x00, + // 1445 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x1b, 0x45, + 0x14, 0x66, 0xbd, 0x76, 0x62, 0x1f, 0xc7, 0xa9, 0x33, 0x6d, 0xd3, 0x6d, 0x40, 0xc1, 0x0c, 0x15, + 0x35, 0x95, 0x1a, 0xaa, 0x16, 0x89, 0x6b, 0xa5, 0x36, 0x71, 0x5a, 0x0c, 0x24, 0x6d, 0xc7, 0x69, + 0xdf, 0x27, 0xeb, 0x51, 0xb3, 0xca, 0x7a, 0xd7, 0xdd, 0x4b, 0xea, 0x14, 0x89, 0x57, 0x10, 0x3c, + 0xf1, 0xc6, 0x03, 0x0f, 0xfc, 0x0f, 0xfe, 0x00, 0x8f, 0xfc, 0x04, 0x54, 0x7e, 0x05, 0x6f, 0x68, + 0xce, 0xcc, 0xec, 0xc5, 0x71, 0x70, 0x68, 0x79, 0x9b, 0x73, 0xff, 0xce, 0x65, 0xce, 0x8e, 0x0d, + 0xad, 0x71, 0xe4, 0x1d, 0xf1, 0x44, 0x6c, 0x8c, 0xa3, 0x30, 0x09, 0x49, 0xdd, 0x0b, 0x12, 0x11, + 0x05, 0xdc, 0x5f, 0x5b, 0x1a, 0xa7, 0xfb, 0xbe, 0xe7, 0x2a, 0x3e, 0xbd, 0x0f, 0x8d, 0x7e, 0x30, + 0x14, 0x93, 0x1d, 0x91, 0x70, 0x42, 0xa0, 0xfa, 0x95, 0x38, 0x8e, 0x1d, 0xbb, 0x63, 0x75, 0xeb, + 0x0c, 0xcf, 0xe4, 0x3d, 0x58, 0xde, 0x8b, 0xb8, 0x7b, 0xb8, 0x3d, 0xf1, 0xe2, 0x44, 0x04, 0xae, + 0x70, 0xaa, 0x28, 0x9d, 0xe2, 0xd2, 0x5f, 0x6d, 0x58, 0xba, 0xe7, 0x09, 0x7f, 0xf8, 0x60, 0x9c, + 0x78, 0x61, 0x10, 0x4b, 0x67, 0x7b, 0xc7, 0x63, 0xe1, 0xd4, 0x3b, 0x56, 0xb7, 0xc1, 0xf0, 0x4c, + 0xde, 0x82, 0xc6, 0x16, 0x77, 0x0f, 0x04, 0x0a, 0x6c, 0x14, 0xe4, 0x8c, 0x4c, 0x3a, 0xf0, 0x5e, + 0xa8, 0x28, 0x2d, 0x96, 0x33, 0x48, 0x07, 0x9a, 0x7b, 0xde, 0x48, 0x3c, 0x4a, 0x79, 0x90, 0xa4, + 0x23, 0xa7, 0x86, 0xd6, 0x45, 0x16, 0x59, 0x85, 0x85, 0x07, 0xfe, 0x70, 0xc7, 0x0b, 0x9c, 0x46, + 0xc7, 0xea, 0xda, 0x4c, 0x53, 0x86, 0xcf, 0x27, 0x0e, 0xe4, 0x7c, 0x3e, 0xc9, 0xd2, 0x6d, 0x96, + 0xd3, 0xdd, 0x0d, 0x07, 0x09, 0x0f, 0x86, 0x3c, 0x1a, 0x3e, 0xf1, 0xc4, 0x73, 0x67, 0x49, 0xa5, + 0x5b, 0xe6, 0x4a, 0xdb, 0x4d, 0x1e, 0x0b, 0xa7, 0x85, 0x1e, 0xf1, 0x4c, 0xd6, 0xa0, 0xbe, 0xe9, + 0x25, 0x3d, 0x31, 0x4e, 0x0e, 0x9c, 0xe5, 0x8e, 0xd5, 0xad, 0xb2, 0x8c, 0x26, 0x17, 0xa0, 0x36, + 0x70, 0xb9, 0x2f, 0x9c, 0x73, 0x68, 0xa0, 0x08, 0x42, 0x61, 0xe9, 0x5e, 0x18, 0x09, 0xef, 0x69, + 0x80, 0x4d, 0x70, 0xda, 0x98, 0x54, 0x89, 0x47, 0xde, 0x05, 0x5b, 0xa6, 0xb4, 0xd2, 0xb1, 0xba, + 0xcd, 0x9b, 0x2b, 0x1b, 0xa6, 0x8f, 0x1b, 0x3d, 0xe1, 0x7a, 0x23, 0xee, 0x33, 0x29, 0x45, 0x25, + 0x3e, 0x71, 0xc8, 0xe9, 0x4a, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x30, 0x4a, 0x98, 0x88, + 0xc7, 0x61, 0x10, 0x0b, 0xd2, 0x06, 0x7b, 0x3b, 0x8a, 0x1c, 0x0b, 0xc3, 0xca, 0x23, 0xfd, 0x16, + 0xda, 0x9b, 0x7e, 0xe8, 0x1e, 0xf6, 0x78, 0xc2, 0x99, 0x78, 0x96, 0x8a, 0x38, 0x91, 0xd8, 0x15, + 0x3c, 0xa5, 0xa7, 0x08, 0xc9, 0xc5, 0x7e, 0x3b, 0x15, 0xc5, 0x45, 0x42, 0xd6, 0x05, 0xab, 0xa6, + 0xda, 0x83, 0x67, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, 0x42, 0x72, 0x31, 0x12, + 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, 0x85, 0x05, 0x16, 0x3e, + 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, 0xe8, 0xa7, 0xa3, 0x40, + 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, 0xb9, 0xad, 0x3c, 0xd2, + 0xef, 0x2c, 0x68, 0xec, 0xf0, 0x09, 0x02, 0x89, 0xc9, 0x6d, 0xa8, 0x9b, 0xde, 0xa2, 0x52, 0xf3, + 0xe6, 0x3b, 0x79, 0x05, 0x33, 0xb5, 0x0d, 0xa3, 0xb3, 0x1d, 0x24, 0xd1, 0x31, 0xcb, 0x4c, 0xd6, + 0x3e, 0x83, 0x56, 0x49, 0x24, 0xe3, 0x1d, 0x8a, 0x63, 0x53, 0xd5, 0x43, 0x71, 0x2c, 0x73, 0x3d, + 0xe2, 0x7e, 0x2a, 0xb0, 0x56, 0x55, 0xa6, 0x88, 0x4f, 0x2b, 0x1f, 0x5b, 0xf4, 0x09, 0x90, 0xad, + 0x48, 0xf0, 0x44, 0x60, 0x90, 0x1d, 0x11, 0xc7, 0xfc, 0xa9, 0x98, 0x57, 0x71, 0xbb, 0x58, 0xf1, + 0xac, 0xba, 0x95, 0x42, 0x75, 0xe9, 0x35, 0x20, 0x3d, 0xe1, 0x8b, 0x44, 0xe8, 0xdb, 0xfd, 0x2f, + 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, 0x30, 0x58, 0xf3, 0xe6, + 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, 0xbb, 0x09, 0x02, 0xb6, + 0x59, 0xce, 0xa0, 0x3f, 0x58, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, 0x93, 0x76, 0x4d, 0x23, + 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, 0x83, 0xb9, 0x63, 0x6a, + 0xf5, 0xaa, 0x58, 0xa8, 0x0b, 0x6f, 0x2a, 0x0f, 0x77, 0x8f, 0xb8, 0xe7, 0xf3, 0x7d, 0xff, 0x3f, + 0xb5, 0xb3, 0x94, 0x96, 0x03, 0x8b, 0x68, 0xdb, 0xef, 0xe9, 0x8b, 0x61, 0x48, 0xfa, 0x0d, 0xe4, + 0x77, 0x6c, 0x97, 0x8f, 0x84, 0xf6, 0x86, 0xe7, 0xac, 0x1a, 0x95, 0x33, 0x54, 0xe3, 0x02, 0xd4, + 0xe4, 0xbd, 0x94, 0x7b, 0xde, 0x96, 0x81, 0x91, 0x98, 0x53, 0xa3, 0x5b, 0xb0, 0x30, 0x70, 0x0f, + 0xc4, 0x88, 0x93, 0xf7, 0x61, 0x11, 0xf1, 0x8b, 0x58, 0x5f, 0x96, 0x73, 0x53, 0x43, 0xc0, 0x8c, + 0x9c, 0xfe, 0x64, 0xe9, 0xc4, 0x67, 0x42, 0x2e, 0x05, 0xac, 0x4c, 0x05, 0x24, 0xd7, 0x61, 0x51, + 0xa3, 0xc6, 0x5d, 0x72, 0xca, 0xac, 0x19, 0x1d, 0x72, 0x15, 0x16, 0x30, 0xd3, 0xd8, 0xa9, 0x4e, + 0x83, 0x42, 0x3e, 0xd3, 0x62, 0xba, 0x0d, 0xf6, 0x63, 0xd6, 0x97, 0x2b, 0x05, 0xf3, 0x31, 0x90, + 0x34, 0x25, 0x81, 0x7e, 0x11, 0xc6, 0x89, 0xee, 0x09, 0x9e, 0x25, 0xef, 0x61, 0x18, 0xa9, 0x29, + 0x6e, 0x31, 0x3c, 0xd3, 0x5f, 0x2c, 0xa8, 0xee, 0x86, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, 0x69, + 0x27, 0x95, 0x7e, 0x8f, 0xbc, 0x8d, 0xfe, 0x75, 0x1f, 0x5a, 0x39, 0x8a, 0xc7, 0xac, 0xcf, 0x30, + 0xf2, 0x15, 0x68, 0xf5, 0xe3, 0xad, 0x30, 0x8c, 0x86, 0x5e, 0xc0, 0x93, 0x30, 0xd2, 0x5f, 0xdb, + 0x32, 0x13, 0x6f, 0x75, 0xc2, 0x13, 0xf5, 0x1d, 0x6c, 0x30, 0x45, 0x90, 0xab, 0xb0, 0x78, 0x9f, + 0x3d, 0xdc, 0x92, 0x01, 0x6a, 0xb3, 0x02, 0x18, 0x29, 0xbd, 0x03, 0x6d, 0x89, 0x0e, 0xad, 0xcc, + 0x14, 0xae, 0xc2, 0x82, 0xe4, 0x65, 0x68, 0x35, 0x95, 0x87, 0xaa, 0x14, 0x42, 0xd1, 0xaf, 0x95, + 0x87, 0xed, 0x23, 0x11, 0x24, 0x85, 0x39, 0x46, 0x1a, 0x1d, 0xb4, 0x98, 0x22, 0x08, 0x55, 0x95, + 0xd0, 0x29, 0x2f, 0xe7, 0x88, 0x24, 0x97, 0xa1, 0x8c, 0xfe, 0x68, 0x01, 0x18, 0x40, 0x69, 0x9c, + 0x99, 0x58, 0xa7, 0x9b, 0x90, 0xae, 0x99, 0x38, 0x7d, 0xc3, 0xdb, 0xb9, 0x96, 0xe2, 0x33, 0x33, + 0x91, 0x1f, 0xe4, 0x13, 0xa9, 0x9a, 0x7f, 0x71, 0x6a, 0x54, 0x54, 0xd4, 0x7c, 0x2e, 0x03, 0x68, + 0x16, 0xf8, 0x33, 0x87, 0xf3, 0x7a, 0x36, 0x4f, 0x95, 0x69, 0x97, 0xc8, 0xd7, 0x2e, 0xb5, 0xd2, + 0x9c, 0x6d, 0xe7, 0x41, 0xb3, 0x60, 0x34, 0x33, 0x5e, 0x17, 0xce, 0x95, 0x77, 0x87, 0xf9, 0xa0, + 0x4d, 0xb3, 0xe7, 0x86, 0x6a, 0x6d, 0xf9, 0x69, 0x9c, 0x88, 0x48, 0x07, 0x93, 0xea, 0x8a, 0x91, + 0x35, 0x3e, 0x67, 0xcc, 0xee, 0x3d, 0xb9, 0x02, 0x35, 0xd9, 0x02, 0xb5, 0x20, 0x4e, 0xf6, 0x47, + 0x09, 0xe9, 0x13, 0xa8, 0x6f, 0x0e, 0xfa, 0xf7, 0xa3, 0x30, 0x1d, 0xcf, 0x4c, 0xc9, 0x3c, 0x00, + 0x2b, 0x85, 0x07, 0x60, 0x5b, 0x3d, 0x66, 0x14, 0x6c, 0x7c, 0xb9, 0xb4, 0xd5, 0xcb, 0xa5, 0xaa, + 0x39, 0x7c, 0x42, 0x07, 0xb0, 0xa2, 0xf2, 0x91, 0x7b, 0xe9, 0x55, 0x56, 0xa8, 0x79, 0x83, 0xd8, + 0xf9, 0x1b, 0x44, 0x3a, 0x55, 0x1b, 0xfa, 0xff, 0x74, 0xfa, 0x77, 0x05, 0x56, 0x98, 0x88, 0xbd, + 0x17, 0xa2, 0x1f, 0xc4, 0x49, 0x94, 0xba, 0x72, 0x17, 0x49, 0xfb, 0x2f, 0xc3, 0x7d, 0x5d, 0x6d, + 0x9b, 0x29, 0xe2, 0x2c, 0xb7, 0x84, 0xdc, 0x80, 0xe6, 0xf4, 0x62, 0x38, 0xa9, 0x5a, 0x54, 0x21, + 0x37, 0x60, 0x71, 0x10, 0xa6, 0x91, 0x9b, 0x8d, 0x7e, 0x61, 0xf3, 0x2b, 0x64, 0x4a, 0xcc, 0x8c, + 0x1a, 0x79, 0x04, 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x9f, 0x3d, + 0x05, 0x9d, 0x92, 0x9f, 0x19, 0xc6, 0xe4, 0xc3, 0xe2, 0xdd, 0x76, 0x16, 0x11, 0xf5, 0x85, 0x32, + 0x6a, 0x7d, 0x5d, 0x8a, 0x3b, 0xe0, 0xf6, 0xd4, 0xa4, 0x3a, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, + 0x89, 0x59, 0x59, 0x9b, 0x7e, 0x6f, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0x76, 0x4a, 0xd6, 0xf0, 0xca, + 0xfc, 0x77, 0x95, 0x69, 0x78, 0x75, 0xd6, 0x4b, 0xb6, 0x56, 0x7c, 0x6b, 0xa5, 0x70, 0xe9, 0x94, + 0x72, 0xbd, 0x06, 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x16, 0xa8, 0xb1, + 0x22, 0x8b, 0x1e, 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0x5f, 0x63, 0x08, + 0xe5, 0x92, 0x8f, 0x22, 0x3d, 0x7e, 0x0d, 0xa6, 0x08, 0xfa, 0x09, 0x5c, 0x1c, 0x88, 0xa4, 0x30, + 0x7a, 0xe6, 0x0e, 0x75, 0xc0, 0xde, 0x15, 0xcf, 0x4f, 0x49, 0x50, 0x8a, 0xe8, 0xe7, 0xe0, 0x3c, + 0x1e, 0x0f, 0x79, 0x22, 0x5e, 0xc9, 0x7a, 0x13, 0xea, 0x7b, 0xe1, 0x38, 0xf4, 0xc3, 0xa7, 0xc7, + 0x73, 0x76, 0x99, 0x03, 0x8b, 0xea, 0x8b, 0xa6, 0x56, 0x67, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xd7, + 0xd4, 0xe5, 0xbe, 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0x51, 0x10, 0x53, 0xa1, 0x2f, 0x02, 0xc7, 0xc2, + 0x15, 0x3e, 0x92, 0x77, 0x91, 0x61, 0x3e, 0x92, 0x8a, 0x22, 0x1f, 0x41, 0xb3, 0xa0, 0xad, 0x0b, + 0x78, 0x71, 0xea, 0xbe, 0x28, 0x21, 0x2b, 0x6a, 0xd2, 0xdf, 0xac, 0x92, 0xe5, 0x89, 0xf7, 0x82, + 0x0e, 0x78, 0xa4, 0x9a, 0x52, 0x67, 0x9a, 0x92, 0xb9, 0x6e, 0x4f, 0x5c, 0x3f, 0x8d, 0xa5, 0x48, + 0x3d, 0x11, 0x72, 0x86, 0xcc, 0x55, 0xfe, 0xf2, 0x0d, 0x53, 0xf3, 0x54, 0x33, 0xa4, 0xfc, 0x11, + 0xda, 0x13, 0x7c, 0xe8, 0x7b, 0x81, 0xc0, 0x29, 0xb5, 0x59, 0x46, 0x93, 0x1b, 0x6a, 0xdb, 0x9b, + 0xab, 0xb6, 0x36, 0x13, 0x3e, 0x6a, 0xa8, 0x2f, 0x41, 0x4c, 0x09, 0xb4, 0xa7, 0x45, 0x9b, 0xed, + 0xdf, 0x5f, 0xae, 0x5b, 0x7f, 0xbc, 0x5c, 0xb7, 0xfe, 0x7c, 0xb9, 0x6e, 0xfd, 0xfc, 0xd7, 0xfa, + 0x1b, 0xfb, 0x0b, 0xf8, 0x5f, 0xc2, 0xad, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x10, 0xd8, 0x2a, + 0xb6, 0x74, 0x10, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3093,8 +3093,8 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -3144,8 +3144,8 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x20 } @@ -3289,8 +3289,8 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x20 } @@ -3416,8 +3416,8 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x10 } @@ -3726,8 +3726,8 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -3779,8 +3779,8 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.ETag != 0 { - i = encodeVarintPrivate(dAtA, i, uint64(m.ETag)) + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) i-- dAtA[i] = 0x18 } @@ -4839,8 +4839,8 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4866,8 +4866,8 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4938,8 +4938,8 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -4975,8 +4975,8 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if len(m.Fields) > 0 { for _, e := range m.Fields { @@ -5129,8 +5129,8 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -5155,8 +5155,8 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } - if m.ETag != 0 { - n += 1 + sovPrivate(uint64(m.ETag)) + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -7121,9 +7121,9 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7133,7 +7133,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7294,9 +7294,9 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7306,7 +7306,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7722,9 +7722,9 @@ func (m *Field) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7734,7 +7734,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -7915,9 +7915,9 @@ func (m *Index) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -7927,7 +7927,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8858,9 +8858,9 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -8870,7 +8870,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -9039,9 +9039,9 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ETag", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) } - m.ETag = 0 + m.CreatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -9051,7 +9051,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ETag |= int64(b&0x7F) << shift + m.CreatedAt |= int64(b&0x7F) << shift if b < 0x80 { break } diff --git a/internal/private.proto b/internal/private.proto index 502d3b7bd..18494a87f 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,14 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; - int64 ETag = 4; + int64 CreatedAt = 4; } message DeleteFieldMessage { @@ -89,7 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; - int64 ETag = 4; + int64 CreatedAt = 4; } message Schema { @@ -98,7 +98,7 @@ message Schema { message Index { string Name = 1; - int64 ETag = 2; + int64 CreatedAt = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -136,13 +136,13 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; - int64 ETag = 3; + int64 CreatedAt = 3; } message ClusterStatus { diff --git a/internal/public.pb.go b/internal/public.pb.go index bf04b2d90..826c6fc62 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1183,6 +1183,8 @@ type ImportRequest struct { RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 { return nil } +func (m *ImportRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1286,6 +1302,8 @@ type ImportValueRequest struct { Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string { return nil } +func (m *ImportValueRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportValueRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct { Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"` Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1726,6 +1760,20 @@ func (m *ImportRoaringRequest) GetBlock() uint64 { return 0 } +func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` @@ -1837,83 +1885,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1207 bytes of a gzipped FileDescriptorProto + // 1253 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b, - 0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b, - 0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00, - 0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d, - 0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa, - 0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a, - 0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b, - 0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e, - 0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf, - 0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92, - 0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a, - 0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85, - 0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23, - 0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42, - 0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3, - 0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5, - 0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87, - 0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54, - 0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b, - 0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87, - 0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a, - 0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e, - 0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36, - 0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75, - 0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72, - 0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae, - 0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c, - 0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde, - 0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68, - 0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2, - 0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c, - 0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47, - 0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0, - 0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3, - 0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e, - 0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17, - 0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97, - 0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61, - 0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb, - 0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29, - 0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef, - 0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c, - 0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30, - 0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea, - 0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0, - 0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5, - 0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c, - 0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d, - 0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b, - 0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43, - 0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6, - 0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b, - 0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c, - 0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac, - 0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2, - 0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9, - 0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1, - 0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67, - 0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe, - 0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90, - 0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e, - 0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc, - 0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d, - 0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9, - 0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6, - 0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed, - 0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42, - 0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde, - 0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6, - 0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc, - 0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6, - 0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf, - 0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7, - 0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc, - 0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00, + 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x91, + 0xb9, 0x21, 0x4d, 0xec, 0x66, 0x33, 0x62, 0x3c, 0x63, 0xe6, 0x07, 0x67, 0x8f, 0x3c, 0x03, 0x17, + 0xc4, 0x13, 0x70, 0xe5, 0x15, 0x38, 0x71, 0xe4, 0x11, 0x50, 0xe0, 0xcc, 0x0b, 0x70, 0x41, 0x55, + 0x3d, 0xed, 0x6e, 0x3b, 0xde, 0x65, 0x15, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xea, 0xeb, 0xea, + 0xaa, 0x1e, 0xe8, 0x2e, 0xca, 0x67, 0x71, 0x34, 0xdd, 0x5f, 0x64, 0x69, 0x91, 0x8a, 0x56, 0x94, + 0x14, 0x32, 0x4b, 0xc2, 0x38, 0xc8, 0xc1, 0xc5, 0x74, 0x29, 0x7c, 0x68, 0x3e, 0x48, 0xe3, 0x72, + 0x9e, 0xe4, 0xbe, 0xd3, 0x77, 0x07, 0x1e, 0x6a, 0x28, 0x04, 0x78, 0x8f, 0xe4, 0x69, 0xee, 0xbb, + 0x7d, 0x77, 0xd0, 0x46, 0x5e, 0x8b, 0x9b, 0x50, 0xbf, 0x5f, 0x14, 0x59, 0xee, 0xd7, 0xfa, 0xee, + 0xa0, 0x73, 0x7b, 0x77, 0x5f, 0xbb, 0xdb, 0x27, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0x34, 0xcc, 0xa2, + 0xe4, 0xc4, 0xf7, 0xfa, 0xce, 0xa0, 0x8b, 0x1a, 0x06, 0x8f, 0xa1, 0x3d, 0x8e, 0x4e, 0x12, 0x39, + 0xa3, 0xad, 0x6f, 0x80, 0xfb, 0x24, 0xa5, 0x6d, 0x9d, 0x41, 0xe7, 0xf6, 0x8e, 0x71, 0x85, 0xe9, + 0x12, 0x49, 0x43, 0x06, 0xc7, 0xf2, 0xc4, 0xaf, 0x6d, 0x35, 0x38, 0x96, 0x27, 0xc1, 0x3d, 0xd8, + 0xc5, 0x74, 0x39, 0x9a, 0xc9, 0xa4, 0x88, 0xbe, 0x8e, 0x64, 0xc6, 0x41, 0x63, 0xba, 0xd4, 0xb9, + 0xf0, 0x7a, 0x95, 0x48, 0xcd, 0x24, 0x12, 0x7c, 0x0c, 0xde, 0x93, 0x30, 0xca, 0xc4, 0x2e, 0xd4, + 0x46, 0x43, 0x0e, 0xc1, 0xc3, 0xda, 0x68, 0x28, 0x2e, 0x83, 0xfb, 0x48, 0x9e, 0xfa, 0x6e, 0xdf, + 0x19, 0xb4, 0x91, 0x96, 0xa2, 0x07, 0xf5, 0x07, 0x69, 0x99, 0x14, 0x1c, 0x86, 0x87, 0x0a, 0x04, + 0x47, 0xd0, 0xa6, 0xef, 0x1f, 0x46, 0x32, 0x9e, 0x89, 0x40, 0x39, 0xab, 0x32, 0xb1, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0x82, 0xcf, 0x01, 0x48, 0x9b, + 0x2b, 0x3f, 0x37, 0xa1, 0xce, 0x88, 0xa3, 0x7f, 0xd5, 0x91, 0x52, 0x9e, 0xe1, 0xe9, 0x1d, 0xa8, + 0x8f, 0x92, 0xe2, 0xee, 0x1d, 0x52, 0x4f, 0xc2, 0xb8, 0x94, 0x1c, 0x8d, 0x8b, 0x0a, 0x04, 0x25, + 0xb4, 0xd8, 0x8e, 0x78, 0x5f, 0x39, 0x70, 0x2c, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x1a, 0x34, 0x30, 0x5d, 0x1a, 0x4a, 0x2a, 0x24, 0xde, 0xd5, 0xbb, 0x78, 0x9c, 0xf3, 0x25, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x0a, 0xe0, 0xb3, 0x2c, 0x2d, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x95, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x6c, 0x27, 0x9d, 0x0e, 0x67, 0x5c, + 0xce, 0x39, 0x12, 0x17, 0x69, 0x19, 0x7c, 0xef, 0x40, 0x6b, 0x12, 0xc6, 0x2b, 0xf5, 0x24, 0x8c, + 0xab, 0xbc, 0x69, 0xb9, 0xee, 0xc6, 0xd5, 0x6e, 0xde, 0x86, 0xd6, 0xc3, 0x38, 0x0d, 0x0b, 0x32, + 0x26, 0x5f, 0x0e, 0xae, 0xb0, 0x38, 0x00, 0x18, 0xca, 0x69, 0x34, 0x0f, 0x63, 0xd2, 0xaa, 0xe4, + 0xde, 0x34, 0x71, 0x56, 0x3a, 0xb4, 0x8c, 0x82, 0x8f, 0xa0, 0x59, 0xa1, 0xed, 0xdc, 0x93, 0x74, + 0x3c, 0x0d, 0x63, 0xa9, 0xa3, 0x60, 0x10, 0x7c, 0x09, 0x3b, 0xea, 0xa6, 0xd1, 0x9d, 0x19, 0xcb, + 0xe2, 0x02, 0xa5, 0x78, 0xa1, 0xdb, 0x17, 0xfc, 0xec, 0x80, 0x47, 0x2b, 0xed, 0xc0, 0x31, 0x0e, + 0x04, 0x78, 0x4f, 0x4f, 0x17, 0xb2, 0x62, 0x95, 0xd7, 0xa2, 0x0f, 0x9d, 0x71, 0x41, 0x97, 0x53, + 0x45, 0xae, 0xb6, 0xb3, 0x45, 0xc4, 0xd7, 0x28, 0x29, 0xcc, 0x71, 0xbb, 0xb8, 0xc2, 0xe2, 0x3a, + 0xb4, 0x0f, 0xd3, 0x34, 0x56, 0xca, 0x7a, 0xdf, 0x19, 0xb4, 0xd0, 0x08, 0xc4, 0x1e, 0x80, 0x66, + 0xb6, 0x94, 0x7e, 0x83, 0xb9, 0xb6, 0x24, 0xc1, 0x2d, 0x68, 0x52, 0xa4, 0x8f, 0xc3, 0x85, 0xc9, + 0xcd, 0x39, 0x2f, 0xb7, 0x7f, 0x1c, 0xe8, 0x7e, 0x51, 0xca, 0xec, 0x14, 0xe5, 0xb7, 0xa5, 0xcc, + 0x0b, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0xe7, 0x61, 0x36, 0x53, 0x4c, 0x79, + 0x58, 0x21, 0xca, 0xd5, 0x70, 0x9e, 0x73, 0xae, 0x2d, 0xb4, 0x45, 0x5c, 0xef, 0x72, 0x9e, 0x16, + 0x3a, 0x99, 0x0a, 0x89, 0x01, 0x5c, 0x3a, 0x7a, 0x31, 0x8d, 0xcb, 0x99, 0xc4, 0x74, 0xa9, 0xbe, + 0x6e, 0xb0, 0xc1, 0xa6, 0x58, 0xbc, 0x07, 0xbb, 0x95, 0x48, 0xf7, 0xd5, 0x26, 0x1b, 0x6e, 0x48, + 0xc5, 0x01, 0x74, 0x8f, 0xe6, 0xcf, 0xe4, 0x6c, 0x26, 0x67, 0xc3, 0xb0, 0x08, 0xfd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0x82, 0x1f, 0x1c, 0xd8, 0xa9, 0xb2, 0xcf, 0x17, 0x69, 0x92, 0x4b, + 0x3a, 0xe2, 0xa3, 0x2c, 0xd3, 0x47, 0x7c, 0x94, 0x65, 0xe2, 0x16, 0x34, 0x51, 0xe6, 0x65, 0x5c, + 0xe8, 0x2a, 0xb9, 0x6a, 0x3c, 0xea, 0x6f, 0xcb, 0xb8, 0x40, 0x6d, 0x25, 0x3e, 0x81, 0xdd, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xfb, 0x2d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0x83, 0xbf, 0x5d, + 0xe8, 0x58, 0x9e, 0x57, 0x45, 0x46, 0xfc, 0xec, 0x54, 0x45, 0x76, 0x83, 0x87, 0xcd, 0x19, 0xad, + 0x9e, 0x7a, 0x52, 0x17, 0x9c, 0xe3, 0xaa, 0x2c, 0x9d, 0x63, 0xd3, 0x08, 0xdd, 0xf3, 0x1a, 0x21, + 0x8d, 0xae, 0xe7, 0x61, 0x72, 0x22, 0x67, 0x5c, 0x96, 0x2d, 0xd4, 0x50, 0xec, 0x9b, 0xae, 0xc0, + 0xe7, 0xb8, 0xd6, 0x6b, 0xb4, 0x06, 0x4d, 0xe7, 0x50, 0x5d, 0x6e, 0x34, 0xa4, 0xb3, 0xe2, 0x7a, + 0x51, 0x48, 0xdc, 0x85, 0x8e, 0x69, 0x5f, 0x79, 0x75, 0x44, 0x3d, 0xe3, 0xca, 0x28, 0xd1, 0x36, + 0x14, 0x9f, 0x6e, 0xce, 0x25, 0xbf, 0xcd, 0x51, 0xf8, 0x6b, 0x99, 0x5b, 0x7a, 0xdc, 0x9c, 0x63, + 0x07, 0xd6, 0xa0, 0xf4, 0x81, 0x3f, 0xbe, 0x62, 0x3e, 0x5e, 0xa9, 0xd0, 0x1a, 0xa7, 0x77, 0xec, + 0x59, 0xe2, 0x77, 0xf8, 0x9b, 0xde, 0x3a, 0x73, 0x4a, 0x87, 0xf6, 0xcc, 0x39, 0xb0, 0x06, 0x99, + 0xdf, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0x15, 0xfc, 0x52, 0x83, 0x9d, 0xd1, 0x7c, 0x91, 0x66, + 0x85, 0x75, 0x0b, 0x47, 0xc9, 0x4c, 0xbe, 0xd0, 0xb7, 0x90, 0xc1, 0xf6, 0x41, 0xc5, 0xdd, 0x90, + 0x6e, 0x23, 0xdf, 0x3e, 0x0f, 0x15, 0xb0, 0x4e, 0xc0, 0x5b, 0x3b, 0x81, 0xeb, 0xd0, 0x56, 0xe5, + 0x46, 0xaa, 0x3a, 0xab, 0x8c, 0x40, 0x3d, 0x34, 0x96, 0x3c, 0xdc, 0x9b, 0x3c, 0xdc, 0x35, 0xa4, + 0xce, 0xa3, 0xcc, 0x58, 0xd9, 0x62, 0xa5, 0x25, 0x21, 0xfd, 0xd3, 0x68, 0x2e, 0xf3, 0x22, 0x9c, + 0x2f, 0xe8, 0x2a, 0xbb, 0x03, 0x17, 0x2d, 0x09, 0xdd, 0x62, 0x4e, 0xe2, 0x41, 0x26, 0xc3, 0x42, + 0xce, 0xee, 0x17, 0x7c, 0x82, 0x2e, 0x6e, 0x48, 0xc9, 0x8e, 0xd3, 0x32, 0x76, 0xa0, 0xec, 0xd6, + 0xa5, 0xc1, 0xaf, 0x35, 0x10, 0x8a, 0x33, 0xee, 0x7c, 0xff, 0x1f, 0x71, 0xe7, 0x13, 0xb4, 0x4e, + 0x43, 0xf3, 0x15, 0x1a, 0xae, 0x41, 0x83, 0xe3, 0xd1, 0x14, 0x54, 0x88, 0x1a, 0xa5, 0x69, 0xd3, + 0x8a, 0x3f, 0x07, 0x6d, 0x91, 0x08, 0xa0, 0x6b, 0xcd, 0x08, 0x2a, 0x70, 0xf2, 0xbd, 0x26, 0xdb, + 0x42, 0x22, 0x5c, 0x90, 0xc4, 0xce, 0x56, 0x12, 0x27, 0xd0, 0x7b, 0x9a, 0x85, 0x49, 0x1e, 0x87, + 0x85, 0xa4, 0xf0, 0x5f, 0x87, 0xc5, 0x2d, 0xaf, 0xda, 0xe0, 0x7d, 0xb8, 0xba, 0xe1, 0xd7, 0xb4, + 0x57, 0xa2, 0xd5, 0x65, 0x5a, 0x69, 0x19, 0x8c, 0xe1, 0xca, 0xca, 0x74, 0x34, 0x7c, 0xad, 0x08, + 0x5e, 0x75, 0xfa, 0x81, 0x95, 0x17, 0x3b, 0xad, 0xb6, 0xdf, 0x16, 0xeb, 0x21, 0xf8, 0xd5, 0xdd, + 0x53, 0x4f, 0xea, 0x2a, 0x82, 0x49, 0x24, 0x97, 0x64, 0x7f, 0x1c, 0xce, 0x65, 0x15, 0x04, 0xaf, + 0x49, 0xc6, 0xe3, 0xa5, 0xc6, 0x0f, 0x71, 0x5e, 0x07, 0x7f, 0x39, 0xd0, 0xdb, 0xe6, 0x84, 0xdf, + 0x4b, 0xb1, 0x0c, 0xd5, 0x40, 0x69, 0xa1, 0x02, 0xe2, 0x1e, 0xd4, 0xbf, 0x8b, 0xe4, 0x52, 0x0f, + 0x94, 0xc0, 0x7a, 0xeb, 0x9d, 0x11, 0x09, 0xaa, 0x0f, 0xa8, 0xbc, 0xee, 0x4f, 0x8b, 0x28, 0x4d, + 0xf4, 0xeb, 0x51, 0x21, 0xda, 0xe7, 0x30, 0x4e, 0xa7, 0xdf, 0x70, 0xdf, 0xf6, 0x50, 0x81, 0x2d, + 0xe5, 0x52, 0xbf, 0x60, 0xb9, 0x34, 0xb6, 0x96, 0xcb, 0x4f, 0x8e, 0xe6, 0xca, 0x9a, 0xf0, 0xff, + 0x79, 0x62, 0xea, 0x8e, 0xe9, 0xa7, 0x1a, 0xdf, 0x31, 0x5f, 0x3d, 0x53, 0xcc, 0x6b, 0x4c, 0x43, + 0x7a, 0x1a, 0xd1, 0x72, 0x12, 0xc6, 0xaa, 0x71, 0xb5, 0x71, 0x85, 0xcf, 0xbf, 0x99, 0x87, 0x97, + 0x7f, 0x7b, 0xb9, 0xe7, 0xfc, 0xfe, 0x72, 0xcf, 0xf9, 0xe3, 0xe5, 0x9e, 0xf3, 0xe3, 0x9f, 0x7b, + 0x6f, 0x3c, 0x6b, 0xf0, 0x1f, 0xda, 0x87, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x91, 0x85, + 0xb9, 0xb1, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -2985,6 +3036,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x50 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x48 + } if len(m.ColumnKeys) > 0 { for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ColumnKeys[iNdEx]) @@ -3104,6 +3165,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x58 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x50 + } if len(m.StringValues) > 0 { for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.StringValues[iNdEx]) @@ -3446,6 +3517,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x30 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x28 + } if m.Block != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.Block)) i-- @@ -4080,6 +4161,12 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4132,6 +4219,12 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4270,6 +4363,12 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPublic(uint64(m.Block)) } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7525,6 +7624,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -7932,6 +8069,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -8771,6 +8946,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 81220fbfe..f1a1fa9fa 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -116,6 +116,7 @@ message QueryResult { message ImportRequest { string Index = 1; + string Field = 2; uint64 Shard = 3; repeated uint64 RowIDs = 4; @@ -123,6 +124,8 @@ message ImportRequest { repeated string RowKeys = 7; repeated string ColumnKeys = 8; repeated int64 Timestamps = 6; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } message ImportValueRequest { @@ -134,6 +137,8 @@ message ImportValueRequest { repeated int64 Values = 6; repeated double FloatValues = 8; repeated string StringValues = 9; + int64 IndexCreatedAt = 10; + int64 FieldCreatedAt = 11; } message TranslateKeysRequest { @@ -166,6 +171,9 @@ message ImportRoaringRequest { repeated ImportRoaringRequestView views = 2; string Action = 3; uint64 Block = 4; + int64 IndexCreatedAt = 5; + int64 FieldCreatedAt = 6; + } message ImportColumnAttrsRequest { diff --git a/pilosa.go b/pilosa.go index e49af5406..060141467 100644 --- a/pilosa.go +++ b/pilosa.go @@ -66,6 +66,9 @@ var ( // we won't need this error at all by 2.0 though. ErrClusterDoesNotOwnShard = errors.New("node does not own shard") + // ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match + ErrPreconditionFailed = errors.New("precondition failed") + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") @@ -125,6 +128,15 @@ func newNotFoundError(err error) NotFoundError { return NotFoundError{err} } +type PreconditionFailedError struct { + error +} + +// newPreconditionFailedError returns err wrapped in a PreconditionFailedError. +func newPreconditionFailedError(err error) PreconditionFailedError { + return PreconditionFailedError{err} +} + // Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) @@ -192,7 +204,7 @@ func stringSlicesAreEqual(a, b []string) bool { return true } -func newETag() int64 { +func timestamp() int64 { return time.Now().UTC().UnixNano() } diff --git a/server.go b/server.go index 586d14e73..8684a6061 100644 --- a/server.go +++ b/server.go @@ -718,7 +718,7 @@ func (s *Server) receiveMessage(m Message) error { return err } idx.mu.Lock() - idx.etag = obj.ETag + idx.createdAt = obj.CreatedAt idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { @@ -735,7 +735,7 @@ func (s *Server) receiveMessage(m Message) error { return err } fld.mu.Lock() - fld.etag = obj.ETag + fld.createdAt = obj.CreatedAt fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) diff --git a/server/handler_test.go b/server/handler_test.go index 9cd25f50e..479856fad 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -195,9 +195,9 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - body := w.Body.String() - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} -`, pilosa.ShardWidth) + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } @@ -213,9 +213,11 @@ func TestHandler_Endpoints(t *testing.T) { idx := indexInfo[0] fld := indexInfo[0].Fields[0] msg := pilosa.ImportRequest{ - Index: idx.Name, - Field: fld.Name, - Shard: 0, + Index: idx.Name, + IndexCreatedAt: idx.CreatedAt, + Field: fld.Name, + FieldCreatedAt: fld.CreatedAt, + Shard: 0, } ser := proto.Serializer{} data, err := ser.Marshal(&msg) @@ -223,29 +225,30 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) - etag := fmt.Sprintf("%d, %d", idx.ETag, fld.ETag) - httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", etag) w := httptest.NewRecorder() h.ServeHTTP(w, httpReq) - if w.Body.String() != "" { + if w.Code != 200 { t.Fatalf(w.Body.String()) } - etag = "invalid-index-etag, invalid-field-etag" - + msg.IndexCreatedAt = -idx.CreatedAt + msg.FieldCreatedAt = -fld.CreatedAt + data, err = ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", etag) + w = httptest.NewRecorder() h.ServeHTTP(w, httpReq) - if strings.TrimSpace(w.Body.String()) != "Precondition Failed" { + if w.Code != 412 { t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) } }) @@ -253,17 +256,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - msg := pilosa.ImportRoaringRequest{ - Clear: false, - Views: map[string][]byte{ - "": roaringData, - }, - } - ser := proto.Serializer{} - data, err := ser.Marshal(&msg) - if err != nil { - t.Fatal(err) - } + idx, err := cmd.API.Index(context.Background(), "i0") if err != nil { t.Fatal(err) @@ -273,12 +266,25 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } + msg := pilosa.ImportRoaringRequest{ + IndexCreatedAt: idx.CreatedAt(), + FieldCreatedAt: fld.CreatedAt(), + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") - httpReq.Header.Set("If-Match", fmt.Sprintf("%d, %d", idx.ETag(), fld.ETag())) h.ServeHTTP(w, httpReq) - if w.Body.String() != "" { + if w.Code != 200 { t.Fatalf("Unexpected response body: %s", w.Body.String()) } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) @@ -821,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected response body: %s", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // Verify index is gone. if hldr.Index("i") != nil { @@ -839,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i").Field("f1"); f != nil { - t.Fatal("expected nil field") + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } } }) @@ -1020,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create index again @@ -1030,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1040,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field again @@ -1050,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // delete field @@ -1060,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete field again @@ -1080,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete index again @@ -1090,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } }) @@ -1102,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1112,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // set some bits From 0c98c887acfd3eb3a1211006186d97bb5d969ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Thu, 28 May 2020 16:28:30 +0200 Subject: [PATCH 3/6] Pass Schema in ClusterStatus message --- api.go | 6 + api_test.go | 46 ++++---- cluster.go | 5 + encoding/proto/proto.go | 15 ++- handler.go | 11 +- holder.go | 26 +++++ http/handler.go | 7 +- internal/private.pb.go | 245 +++++++++++++++++++++++++--------------- internal/private.proto | 1 + internal/public.pb.go | 191 ++++++++++++++++++------------- internal/public.proto | 1 + 11 files changed, 351 insertions(+), 203 deletions(-) diff --git a/api.go b/api.go index 0a29ad30a..54bb8f3be 100644 --- a/api.go +++ b/api.go @@ -1302,6 +1302,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq return errors.Wrap(err, "validating shard ownership") } + if req.IndexCreatedAt != 0 { + if index.CreatedAt() != req.IndexCreatedAt { + return ErrPreconditionFailed + } + } + bulkAttrs := make(map[uint64]map[string]interface{}) for n := 0; n < len(req.ColumnIDs); n++ { bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} diff --git a/api_test.go b/api_test.go index e8b4f28f5..02e8a182f 100644 --- a/api_test.go +++ b/api_test.go @@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { m1 := c[1] t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() - index := "i" - field := "f" + indexName := "i" + fieldName := "f" attrKey := "k" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field) + _, err = m0.API.CreateField(ctx, indexName, fieldName) if err != nil { t.Fatalf("creating field: %v", err) } @@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { columnIDs0 = append(columnIDs0, uint64(n)) val0 := attrFun(uint64(n)) attrVals0 = append(attrVals0, val0) - setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field) - if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil { + setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName) + if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil { t.Fatal(err) } columnIDs1 = append(columnIDs1, uint64(n+ShardWidth)) val1 := attrFun(uint64(n + ShardWidth)) attrVals1 = append(attrVals1, val1) - setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field) - if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil { + setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName) + if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil { t.Fatal(err) } } // send shard0 to node1 req := &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs0, - AttrVals: attrVals0, - Shard: 0, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs0, + AttrVals: attrVals0, + Shard: 0, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { @@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { // send shard1 to node0 req = &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs1, - AttrVals: attrVals1, - Shard: 1, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs1, + AttrVals: attrVals1, + Shard: 1, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { @@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } // Query node0. - pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } } // Query node1. - pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } diff --git a/cluster.go b/cluster.go index d283593a1..ced55e67a 100644 --- a/cluster.go +++ b/cluster.go @@ -610,6 +610,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, + Schema: &Schema{Indexes: c.holder.Schema()}, } } @@ -2213,6 +2214,9 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } + if cs.Schema != nil { + c.holder.applyCreatedAt(cs.Schema.Indexes) + } c.unprotectedSetState(cs.State) c.markAsJoined() @@ -2449,6 +2453,7 @@ type ClusterStatus struct { ClusterID string State string Nodes []*Node + Schema *Schema } // ResizeInstruction contains the instruction provided to a node diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 7d647ded0..82a31ce05 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -453,11 +453,12 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest { return &internal.ImportColumnAttrsRequest{ - Index: m.Index, - Shard: m.Shard, - AttrKey: m.AttrKey, - AttrVals: m.AttrVals, - ColumnIDs: m.ColumnIDs, + Index: m.Index, + IndexCreatedAt: m.IndexCreatedAt, + Shard: m.Shard, + AttrKey: m.AttrKey, + AttrVals: m.AttrVals, + ColumnIDs: m.ColumnIDs, } } @@ -680,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust State: m.State, ClusterID: m.ClusterID, Nodes: s.encodeNodes(m.Nodes), + Schema: s.encodeSchema(m.Schema), } } @@ -1006,6 +1008,8 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl m.ClusterID = cs.ClusterID m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) s.decodeNodes(cs.Nodes, m.Nodes) + m.Schema = &pilosa.Schema{} + s.decodeSchema(cs.Schema, m.Schema) } func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { @@ -1199,6 +1203,7 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { m.Index = pb.Index + m.IndexCreatedAt = pb.IndexCreatedAt m.Shard = pb.Shard m.AttrKey = pb.AttrKey m.AttrVals = pb.AttrVals diff --git a/handler.go b/handler.go index 79c60e285..17156245b 100644 --- a/handler.go +++ b/handler.go @@ -178,11 +178,12 @@ func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreate // ImportColumnAttrsRequest describes the import request structure // for a ColumnAttr import type ImportColumnAttrsRequest struct { - AttrKey string - ColumnIDs []uint64 - AttrVals []string - Shard int64 - Index string + AttrKey string + ColumnIDs []uint64 + AttrVals []string + Shard int64 + Index string + IndexCreatedAt int64 } // ImportRequest describes the import request structure diff --git a/holder.go b/holder.go index dd995566a..70501a4b1 100644 --- a/holder.go +++ b/holder.go @@ -475,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } +func (h *Holder) applyCreatedAt(indexes []*IndexInfo) { + for _, ii := range indexes { + idx := h.Index(ii.Name) + if idx == nil { + continue + } + if ii.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = ii.CreatedAt + idx.mu.Unlock() + } + + for _, fi := range ii.Fields { + fld := idx.Field(fi.Name) + if fld == nil { + continue + } + if fi.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = fi.CreatedAt + fld.mu.Unlock() + } + } + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } diff --git a/http/handler.go b/http/handler.go index e2787d77c..12773f3ee 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1935,7 +1935,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } diff --git a/internal/private.pb.go b/internal/private.pb.go index ea281b409..6dd3eb9bc 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1499,6 +1499,7 @@ type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1558,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } +func (m *ClusterStatus) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -2469,98 +2477,99 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1445 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x6f, 0x1b, 0x45, - 0x14, 0x66, 0xbd, 0x76, 0x62, 0x1f, 0xc7, 0xa9, 0x33, 0x6d, 0xd3, 0x6d, 0x40, 0xc1, 0x0c, 0x15, - 0x35, 0x95, 0x1a, 0xaa, 0x16, 0x89, 0x6b, 0xa5, 0x36, 0x71, 0x5a, 0x0c, 0x24, 0x6d, 0xc7, 0x69, - 0xdf, 0x27, 0xeb, 0x51, 0xb3, 0xca, 0x7a, 0xd7, 0xdd, 0x4b, 0xea, 0x14, 0x89, 0x57, 0x10, 0x3c, - 0xf1, 0xc6, 0x03, 0x0f, 0xfc, 0x0f, 0xfe, 0x00, 0x8f, 0xfc, 0x04, 0x54, 0x7e, 0x05, 0x6f, 0x68, - 0xce, 0xcc, 0xec, 0xc5, 0x71, 0x70, 0x68, 0x79, 0x9b, 0x73, 0xff, 0xce, 0x65, 0xce, 0x8e, 0x0d, - 0xad, 0x71, 0xe4, 0x1d, 0xf1, 0x44, 0x6c, 0x8c, 0xa3, 0x30, 0x09, 0x49, 0xdd, 0x0b, 0x12, 0x11, - 0x05, 0xdc, 0x5f, 0x5b, 0x1a, 0xa7, 0xfb, 0xbe, 0xe7, 0x2a, 0x3e, 0xbd, 0x0f, 0x8d, 0x7e, 0x30, - 0x14, 0x93, 0x1d, 0x91, 0x70, 0x42, 0xa0, 0xfa, 0x95, 0x38, 0x8e, 0x1d, 0xbb, 0x63, 0x75, 0xeb, - 0x0c, 0xcf, 0xe4, 0x3d, 0x58, 0xde, 0x8b, 0xb8, 0x7b, 0xb8, 0x3d, 0xf1, 0xe2, 0x44, 0x04, 0xae, - 0x70, 0xaa, 0x28, 0x9d, 0xe2, 0xd2, 0x5f, 0x6d, 0x58, 0xba, 0xe7, 0x09, 0x7f, 0xf8, 0x60, 0x9c, - 0x78, 0x61, 0x10, 0x4b, 0x67, 0x7b, 0xc7, 0x63, 0xe1, 0xd4, 0x3b, 0x56, 0xb7, 0xc1, 0xf0, 0x4c, - 0xde, 0x82, 0xc6, 0x16, 0x77, 0x0f, 0x04, 0x0a, 0x6c, 0x14, 0xe4, 0x8c, 0x4c, 0x3a, 0xf0, 0x5e, - 0xa8, 0x28, 0x2d, 0x96, 0x33, 0x48, 0x07, 0x9a, 0x7b, 0xde, 0x48, 0x3c, 0x4a, 0x79, 0x90, 0xa4, - 0x23, 0xa7, 0x86, 0xd6, 0x45, 0x16, 0x59, 0x85, 0x85, 0x07, 0xfe, 0x70, 0xc7, 0x0b, 0x9c, 0x46, - 0xc7, 0xea, 0xda, 0x4c, 0x53, 0x86, 0xcf, 0x27, 0x0e, 0xe4, 0x7c, 0x3e, 0xc9, 0xd2, 0x6d, 0x96, - 0xd3, 0xdd, 0x0d, 0x07, 0x09, 0x0f, 0x86, 0x3c, 0x1a, 0x3e, 0xf1, 0xc4, 0x73, 0x67, 0x49, 0xa5, - 0x5b, 0xe6, 0x4a, 0xdb, 0x4d, 0x1e, 0x0b, 0xa7, 0x85, 0x1e, 0xf1, 0x4c, 0xd6, 0xa0, 0xbe, 0xe9, - 0x25, 0x3d, 0x31, 0x4e, 0x0e, 0x9c, 0xe5, 0x8e, 0xd5, 0xad, 0xb2, 0x8c, 0x26, 0x17, 0xa0, 0x36, - 0x70, 0xb9, 0x2f, 0x9c, 0x73, 0x68, 0xa0, 0x08, 0x42, 0x61, 0xe9, 0x5e, 0x18, 0x09, 0xef, 0x69, - 0x80, 0x4d, 0x70, 0xda, 0x98, 0x54, 0x89, 0x47, 0xde, 0x05, 0x5b, 0xa6, 0xb4, 0xd2, 0xb1, 0xba, - 0xcd, 0x9b, 0x2b, 0x1b, 0xa6, 0x8f, 0x1b, 0x3d, 0xe1, 0x7a, 0x23, 0xee, 0x33, 0x29, 0x45, 0x25, - 0x3e, 0x71, 0xc8, 0xe9, 0x4a, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x30, 0x4a, 0x98, 0x88, - 0xc7, 0x61, 0x10, 0x0b, 0xd2, 0x06, 0x7b, 0x3b, 0x8a, 0x1c, 0x0b, 0xc3, 0xca, 0x23, 0xfd, 0x16, - 0xda, 0x9b, 0x7e, 0xe8, 0x1e, 0xf6, 0x78, 0xc2, 0x99, 0x78, 0x96, 0x8a, 0x38, 0x91, 0xd8, 0x15, - 0x3c, 0xa5, 0xa7, 0x08, 0xc9, 0xc5, 0x7e, 0x3b, 0x15, 0xc5, 0x45, 0x42, 0xd6, 0x05, 0xab, 0xa6, - 0xda, 0x83, 0x67, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, 0x42, 0x72, 0x31, 0x12, - 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, 0x85, 0x05, 0x16, 0x3e, - 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, 0xe8, 0xa7, 0xa3, 0x40, - 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, 0xb9, 0xad, 0x3c, 0xd2, - 0xef, 0x2c, 0x68, 0xec, 0xf0, 0x09, 0x02, 0x89, 0xc9, 0x6d, 0xa8, 0x9b, 0xde, 0xa2, 0x52, 0xf3, - 0xe6, 0x3b, 0x79, 0x05, 0x33, 0xb5, 0x0d, 0xa3, 0xb3, 0x1d, 0x24, 0xd1, 0x31, 0xcb, 0x4c, 0xd6, - 0x3e, 0x83, 0x56, 0x49, 0x24, 0xe3, 0x1d, 0x8a, 0x63, 0x53, 0xd5, 0x43, 0x71, 0x2c, 0x73, 0x3d, - 0xe2, 0x7e, 0x2a, 0xb0, 0x56, 0x55, 0xa6, 0x88, 0x4f, 0x2b, 0x1f, 0x5b, 0xf4, 0x09, 0x90, 0xad, - 0x48, 0xf0, 0x44, 0x60, 0x90, 0x1d, 0x11, 0xc7, 0xfc, 0xa9, 0x98, 0x57, 0x71, 0xbb, 0x58, 0xf1, - 0xac, 0xba, 0x95, 0x42, 0x75, 0xe9, 0x35, 0x20, 0x3d, 0xe1, 0x8b, 0x44, 0xe8, 0xdb, 0xfd, 0x2f, - 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, 0x30, 0x58, 0xf3, 0xe6, - 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, 0xbb, 0x09, 0x02, 0xb6, - 0x59, 0xce, 0xa0, 0x3f, 0x58, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, 0x93, 0x76, 0x4d, 0x23, - 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, 0x83, 0xb9, 0x63, 0x6a, - 0xf5, 0xaa, 0x58, 0xa8, 0x0b, 0x6f, 0x2a, 0x0f, 0x77, 0x8f, 0xb8, 0xe7, 0xf3, 0x7d, 0xff, 0x3f, - 0xb5, 0xb3, 0x94, 0x96, 0x03, 0x8b, 0x68, 0xdb, 0xef, 0xe9, 0x8b, 0x61, 0x48, 0xfa, 0x0d, 0xe4, - 0x77, 0x6c, 0x97, 0x8f, 0x84, 0xf6, 0x86, 0xe7, 0xac, 0x1a, 0x95, 0x33, 0x54, 0xe3, 0x02, 0xd4, - 0xe4, 0xbd, 0x94, 0x7b, 0xde, 0x96, 0x81, 0x91, 0x98, 0x53, 0xa3, 0x5b, 0xb0, 0x30, 0x70, 0x0f, - 0xc4, 0x88, 0x93, 0xf7, 0x61, 0x11, 0xf1, 0x8b, 0x58, 0x5f, 0x96, 0x73, 0x53, 0x43, 0xc0, 0x8c, - 0x9c, 0xfe, 0x64, 0xe9, 0xc4, 0x67, 0x42, 0x2e, 0x05, 0xac, 0x4c, 0x05, 0x24, 0xd7, 0x61, 0x51, - 0xa3, 0xc6, 0x5d, 0x72, 0xca, 0xac, 0x19, 0x1d, 0x72, 0x15, 0x16, 0x30, 0xd3, 0xd8, 0xa9, 0x4e, - 0x83, 0x42, 0x3e, 0xd3, 0x62, 0xba, 0x0d, 0xf6, 0x63, 0xd6, 0x97, 0x2b, 0x05, 0xf3, 0x31, 0x90, - 0x34, 0x25, 0x81, 0x7e, 0x11, 0xc6, 0x89, 0xee, 0x09, 0x9e, 0x25, 0xef, 0x61, 0x18, 0xa9, 0x29, - 0x6e, 0x31, 0x3c, 0xd3, 0x5f, 0x2c, 0xa8, 0xee, 0x86, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, 0x69, - 0x27, 0x95, 0x7e, 0x8f, 0xbc, 0x8d, 0xfe, 0x75, 0x1f, 0x5a, 0x39, 0x8a, 0xc7, 0xac, 0xcf, 0x30, - 0xf2, 0x15, 0x68, 0xf5, 0xe3, 0xad, 0x30, 0x8c, 0x86, 0x5e, 0xc0, 0x93, 0x30, 0xd2, 0x5f, 0xdb, - 0x32, 0x13, 0x6f, 0x75, 0xc2, 0x13, 0xf5, 0x1d, 0x6c, 0x30, 0x45, 0x90, 0xab, 0xb0, 0x78, 0x9f, - 0x3d, 0xdc, 0x92, 0x01, 0x6a, 0xb3, 0x02, 0x18, 0x29, 0xbd, 0x03, 0x6d, 0x89, 0x0e, 0xad, 0xcc, - 0x14, 0xae, 0xc2, 0x82, 0xe4, 0x65, 0x68, 0x35, 0x95, 0x87, 0xaa, 0x14, 0x42, 0xd1, 0xaf, 0x95, - 0x87, 0xed, 0x23, 0x11, 0x24, 0x85, 0x39, 0x46, 0x1a, 0x1d, 0xb4, 0x98, 0x22, 0x08, 0x55, 0x95, - 0xd0, 0x29, 0x2f, 0xe7, 0x88, 0x24, 0x97, 0xa1, 0x8c, 0xfe, 0x68, 0x01, 0x18, 0x40, 0x69, 0x9c, - 0x99, 0x58, 0xa7, 0x9b, 0x90, 0xae, 0x99, 0x38, 0x7d, 0xc3, 0xdb, 0xb9, 0x96, 0xe2, 0x33, 0x33, - 0x91, 0x1f, 0xe4, 0x13, 0xa9, 0x9a, 0x7f, 0x71, 0x6a, 0x54, 0x54, 0xd4, 0x7c, 0x2e, 0x03, 0x68, - 0x16, 0xf8, 0x33, 0x87, 0xf3, 0x7a, 0x36, 0x4f, 0x95, 0x69, 0x97, 0xc8, 0xd7, 0x2e, 0xb5, 0xd2, - 0x9c, 0x6d, 0xe7, 0x41, 0xb3, 0x60, 0x34, 0x33, 0x5e, 0x17, 0xce, 0x95, 0x77, 0x87, 0xf9, 0xa0, - 0x4d, 0xb3, 0xe7, 0x86, 0x6a, 0x6d, 0xf9, 0x69, 0x9c, 0x88, 0x48, 0x07, 0x93, 0xea, 0x8a, 0x91, - 0x35, 0x3e, 0x67, 0xcc, 0xee, 0x3d, 0xb9, 0x02, 0x35, 0xd9, 0x02, 0xb5, 0x20, 0x4e, 0xf6, 0x47, - 0x09, 0xe9, 0x13, 0xa8, 0x6f, 0x0e, 0xfa, 0xf7, 0xa3, 0x30, 0x1d, 0xcf, 0x4c, 0xc9, 0x3c, 0x00, - 0x2b, 0x85, 0x07, 0x60, 0x5b, 0x3d, 0x66, 0x14, 0x6c, 0x7c, 0xb9, 0xb4, 0xd5, 0xcb, 0xa5, 0xaa, - 0x39, 0x7c, 0x42, 0x07, 0xb0, 0xa2, 0xf2, 0x91, 0x7b, 0xe9, 0x55, 0x56, 0xa8, 0x79, 0x83, 0xd8, - 0xf9, 0x1b, 0x44, 0x3a, 0x55, 0x1b, 0xfa, 0xff, 0x74, 0xfa, 0x77, 0x05, 0x56, 0x98, 0x88, 0xbd, - 0x17, 0xa2, 0x1f, 0xc4, 0x49, 0x94, 0xba, 0x72, 0x17, 0x49, 0xfb, 0x2f, 0xc3, 0x7d, 0x5d, 0x6d, - 0x9b, 0x29, 0xe2, 0x2c, 0xb7, 0x84, 0xdc, 0x80, 0xe6, 0xf4, 0x62, 0x38, 0xa9, 0x5a, 0x54, 0x21, - 0x37, 0x60, 0x71, 0x10, 0xa6, 0x91, 0x9b, 0x8d, 0x7e, 0x61, 0xf3, 0x2b, 0x64, 0x4a, 0xcc, 0x8c, - 0x1a, 0x79, 0x04, 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x9f, 0x3d, - 0x05, 0x9d, 0x92, 0x9f, 0x19, 0xc6, 0xe4, 0xc3, 0xe2, 0xdd, 0x76, 0x16, 0x11, 0xf5, 0x85, 0x32, - 0x6a, 0x7d, 0x5d, 0x8a, 0x3b, 0xe0, 0xf6, 0xd4, 0xa4, 0x3a, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, - 0x89, 0x59, 0x59, 0x9b, 0x7e, 0x6f, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0x76, 0x4a, 0xd6, 0xf0, 0xca, - 0xfc, 0x77, 0x95, 0x69, 0x78, 0x75, 0xd6, 0x4b, 0xb6, 0x56, 0x7c, 0x6b, 0xa5, 0x70, 0xe9, 0x94, - 0x72, 0xbd, 0x06, 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x16, 0xa8, 0xb1, - 0x22, 0x8b, 0x1e, 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0x5f, 0x63, 0x08, - 0xe5, 0x92, 0x8f, 0x22, 0x3d, 0x7e, 0x0d, 0xa6, 0x08, 0xfa, 0x09, 0x5c, 0x1c, 0x88, 0xa4, 0x30, - 0x7a, 0xe6, 0x0e, 0x75, 0xc0, 0xde, 0x15, 0xcf, 0x4f, 0x49, 0x50, 0x8a, 0xe8, 0xe7, 0xe0, 0x3c, - 0x1e, 0x0f, 0x79, 0x22, 0x5e, 0xc9, 0x7a, 0x13, 0xea, 0x7b, 0xe1, 0x38, 0xf4, 0xc3, 0xa7, 0xc7, - 0x73, 0x76, 0x99, 0x03, 0x8b, 0xea, 0x8b, 0xa6, 0x56, 0x67, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xd7, - 0xd4, 0xe5, 0xbe, 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0x51, 0x10, 0x53, 0xa1, 0x2f, 0x02, 0xc7, 0xc2, - 0x15, 0x3e, 0x92, 0x77, 0x91, 0x61, 0x3e, 0x92, 0x8a, 0x22, 0x1f, 0x41, 0xb3, 0xa0, 0xad, 0x0b, - 0x78, 0x71, 0xea, 0xbe, 0x28, 0x21, 0x2b, 0x6a, 0xd2, 0xdf, 0xac, 0x92, 0xe5, 0x89, 0xf7, 0x82, - 0x0e, 0x78, 0xa4, 0x9a, 0x52, 0x67, 0x9a, 0x92, 0xb9, 0x6e, 0x4f, 0x5c, 0x3f, 0x8d, 0xa5, 0x48, - 0x3d, 0x11, 0x72, 0x86, 0xcc, 0x55, 0xfe, 0xf2, 0x0d, 0x53, 0xf3, 0x54, 0x33, 0xa4, 0xfc, 0x11, - 0xda, 0x13, 0x7c, 0xe8, 0x7b, 0x81, 0xc0, 0x29, 0xb5, 0x59, 0x46, 0x93, 0x1b, 0x6a, 0xdb, 0x9b, - 0xab, 0xb6, 0x36, 0x13, 0x3e, 0x6a, 0xa8, 0x2f, 0x41, 0x4c, 0x09, 0xb4, 0xa7, 0x45, 0x9b, 0xed, - 0xdf, 0x5f, 0xae, 0x5b, 0x7f, 0xbc, 0x5c, 0xb7, 0xfe, 0x7c, 0xb9, 0x6e, 0xfd, 0xfc, 0xd7, 0xfa, - 0x1b, 0xfb, 0x0b, 0xf8, 0x5f, 0xc2, 0xad, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x10, 0xd8, 0x2a, - 0xb6, 0x74, 0x10, 0x00, 0x00, + // 1458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45, + 0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf, + 0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0, + 0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa, + 0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0, + 0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97, + 0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75, + 0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e, + 0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec, + 0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17, + 0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8, + 0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba, + 0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91, + 0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47, + 0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee, + 0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27, + 0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78, + 0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93, + 0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3, + 0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17, + 0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29, + 0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb, + 0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38, + 0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0, + 0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45, + 0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21, + 0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, + 0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, + 0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, + 0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, + 0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37, + 0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2, + 0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87, + 0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4, + 0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf, + 0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89, + 0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, + 0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, + 0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, + 0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, + 0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf, + 0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3, + 0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce, + 0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e, + 0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce, + 0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15, + 0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0, + 0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae, + 0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79, + 0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64, + 0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47, + 0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78, + 0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82, + 0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68, + 0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5, + 0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5, + 0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b, + 0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce, + 0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2, + 0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44, + 0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae, + 0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27, + 0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d, + 0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02, + 0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56, + 0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0, + 0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91, + 0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07, + 0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38, + 0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c, + 0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04, + 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92, + 0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2, + 0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59, + 0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98, + 0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01, + 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e, + 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10, + 0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19, + 0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2, + 0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d, + 0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe, + 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3, + 0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a, + 0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9, + 0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c, + 0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f, + 0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b, + 0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb, + 0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05, + 0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3836,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Schema != nil { + { + size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if len(m.Nodes) > 0 { for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { { @@ -5184,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9208,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 18494a87f..d83a8d2c0 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -149,6 +149,7 @@ message ClusterStatus { string ClusterID = 1; string State = 2; repeated Node Nodes = 3; + Schema Schema = 4; } message BSIGroup { diff --git a/internal/public.pb.go b/internal/public.pb.go index 826c6fc62..9f71b0368 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1780,6 +1780,7 @@ type ImportColumnAttrsRequest struct { AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"` AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1853,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 { return nil } +func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") @@ -1885,86 +1893,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1253 bytes of a gzipped FileDescriptorProto + // 1258 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, - 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x91, - 0xb9, 0x21, 0x4d, 0xec, 0x66, 0x33, 0x62, 0x3c, 0x63, 0xe6, 0x07, 0x67, 0x8f, 0x3c, 0x03, 0x17, - 0xc4, 0x13, 0x70, 0xe5, 0x15, 0x38, 0x71, 0xe4, 0x11, 0x50, 0xe0, 0xcc, 0x0b, 0x70, 0x41, 0x55, - 0x3d, 0xed, 0x6e, 0x3b, 0xde, 0x65, 0x15, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xea, 0xeb, 0xea, - 0xaa, 0x1e, 0xe8, 0x2e, 0xca, 0x67, 0x71, 0x34, 0xdd, 0x5f, 0x64, 0x69, 0x91, 0x8a, 0x56, 0x94, - 0x14, 0x32, 0x4b, 0xc2, 0x38, 0xc8, 0xc1, 0xc5, 0x74, 0x29, 0x7c, 0x68, 0x3e, 0x48, 0xe3, 0x72, - 0x9e, 0xe4, 0xbe, 0xd3, 0x77, 0x07, 0x1e, 0x6a, 0x28, 0x04, 0x78, 0x8f, 0xe4, 0x69, 0xee, 0xbb, - 0x7d, 0x77, 0xd0, 0x46, 0x5e, 0x8b, 0x9b, 0x50, 0xbf, 0x5f, 0x14, 0x59, 0xee, 0xd7, 0xfa, 0xee, - 0xa0, 0x73, 0x7b, 0x77, 0x5f, 0xbb, 0xdb, 0x27, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0x34, 0xcc, 0xa2, - 0xe4, 0xc4, 0xf7, 0xfa, 0xce, 0xa0, 0x8b, 0x1a, 0x06, 0x8f, 0xa1, 0x3d, 0x8e, 0x4e, 0x12, 0x39, - 0xa3, 0xad, 0x6f, 0x80, 0xfb, 0x24, 0xa5, 0x6d, 0x9d, 0x41, 0xe7, 0xf6, 0x8e, 0x71, 0x85, 0xe9, - 0x12, 0x49, 0x43, 0x06, 0xc7, 0xf2, 0xc4, 0xaf, 0x6d, 0x35, 0x38, 0x96, 0x27, 0xc1, 0x3d, 0xd8, - 0xc5, 0x74, 0x39, 0x9a, 0xc9, 0xa4, 0x88, 0xbe, 0x8e, 0x64, 0xc6, 0x41, 0x63, 0xba, 0xd4, 0xb9, - 0xf0, 0x7a, 0x95, 0x48, 0xcd, 0x24, 0x12, 0x7c, 0x0c, 0xde, 0x93, 0x30, 0xca, 0xc4, 0x2e, 0xd4, - 0x46, 0x43, 0x0e, 0xc1, 0xc3, 0xda, 0x68, 0x28, 0x2e, 0x83, 0xfb, 0x48, 0x9e, 0xfa, 0x6e, 0xdf, - 0x19, 0xb4, 0x91, 0x96, 0xa2, 0x07, 0xf5, 0x07, 0x69, 0x99, 0x14, 0x1c, 0x86, 0x87, 0x0a, 0x04, - 0x47, 0xd0, 0xa6, 0xef, 0x1f, 0x46, 0x32, 0x9e, 0x89, 0x40, 0x39, 0xab, 0x32, 0xb1, 0x48, 0x21, - 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0x82, 0xcf, 0x01, 0x48, 0x9b, - 0x2b, 0x3f, 0x37, 0xa1, 0xce, 0x88, 0xa3, 0x7f, 0xd5, 0x91, 0x52, 0x9e, 0xe1, 0xe9, 0x1d, 0xa8, - 0x8f, 0x92, 0xe2, 0xee, 0x1d, 0x52, 0x4f, 0xc2, 0xb8, 0x94, 0x1c, 0x8d, 0x8b, 0x0a, 0x04, 0x25, - 0xb4, 0xd8, 0x8e, 0x78, 0x5f, 0x39, 0x70, 0x2c, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, - 0xe2, 0x1a, 0x34, 0x30, 0x5d, 0x1a, 0x4a, 0x2a, 0x24, 0xde, 0xd5, 0xbb, 0x78, 0x9c, 0xf3, 0x25, - 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x0a, 0xe0, 0xb3, 0x2c, 0x2d, 0x17, 0x4c, 0x9a, 0x18, 0x40, - 0x9d, 0x51, 0x95, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x6c, 0x27, 0x9d, 0x0e, 0x67, 0x5c, - 0xce, 0x39, 0x12, 0x17, 0x69, 0x19, 0x7c, 0xef, 0x40, 0x6b, 0x12, 0xc6, 0x2b, 0xf5, 0x24, 0x8c, - 0xab, 0xbc, 0x69, 0xb9, 0xee, 0xc6, 0xd5, 0x6e, 0xde, 0x86, 0xd6, 0xc3, 0x38, 0x0d, 0x0b, 0x32, - 0x26, 0x5f, 0x0e, 0xae, 0xb0, 0x38, 0x00, 0x18, 0xca, 0x69, 0x34, 0x0f, 0x63, 0xd2, 0xaa, 0xe4, - 0xde, 0x34, 0x71, 0x56, 0x3a, 0xb4, 0x8c, 0x82, 0x8f, 0xa0, 0x59, 0xa1, 0xed, 0xdc, 0x93, 0x74, - 0x3c, 0x0d, 0x63, 0xa9, 0xa3, 0x60, 0x10, 0x7c, 0x09, 0x3b, 0xea, 0xa6, 0xd1, 0x9d, 0x19, 0xcb, - 0xe2, 0x02, 0xa5, 0x78, 0xa1, 0xdb, 0x17, 0xfc, 0xec, 0x80, 0x47, 0x2b, 0xed, 0xc0, 0x31, 0x0e, - 0x04, 0x78, 0x4f, 0x4f, 0x17, 0xb2, 0x62, 0x95, 0xd7, 0xa2, 0x0f, 0x9d, 0x71, 0x41, 0x97, 0x53, - 0x45, 0xae, 0xb6, 0xb3, 0x45, 0xc4, 0xd7, 0x28, 0x29, 0xcc, 0x71, 0xbb, 0xb8, 0xc2, 0xe2, 0x3a, - 0xb4, 0x0f, 0xd3, 0x34, 0x56, 0xca, 0x7a, 0xdf, 0x19, 0xb4, 0xd0, 0x08, 0xc4, 0x1e, 0x80, 0x66, - 0xb6, 0x94, 0x7e, 0x83, 0xb9, 0xb6, 0x24, 0xc1, 0x2d, 0x68, 0x52, 0xa4, 0x8f, 0xc3, 0x85, 0xc9, - 0xcd, 0x39, 0x2f, 0xb7, 0x7f, 0x1c, 0xe8, 0x7e, 0x51, 0xca, 0xec, 0x14, 0xe5, 0xb7, 0xa5, 0xcc, - 0x0b, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0xe7, 0x61, 0x36, 0x53, 0x4c, 0x79, - 0x58, 0x21, 0xca, 0xd5, 0x70, 0x9e, 0x73, 0xae, 0x2d, 0xb4, 0x45, 0x5c, 0xef, 0x72, 0x9e, 0x16, - 0x3a, 0x99, 0x0a, 0x89, 0x01, 0x5c, 0x3a, 0x7a, 0x31, 0x8d, 0xcb, 0x99, 0xc4, 0x74, 0xa9, 0xbe, - 0x6e, 0xb0, 0xc1, 0xa6, 0x58, 0xbc, 0x07, 0xbb, 0x95, 0x48, 0xf7, 0xd5, 0x26, 0x1b, 0x6e, 0x48, - 0xc5, 0x01, 0x74, 0x8f, 0xe6, 0xcf, 0xe4, 0x6c, 0x26, 0x67, 0xc3, 0xb0, 0x08, 0xfd, 0x16, 0xe7, - 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0x82, 0x1f, 0x1c, 0xd8, 0xa9, 0xb2, 0xcf, 0x17, 0x69, 0x92, 0x4b, - 0x3a, 0xe2, 0xa3, 0x2c, 0xd3, 0x47, 0x7c, 0x94, 0x65, 0xe2, 0x16, 0x34, 0x51, 0xe6, 0x65, 0x5c, - 0xe8, 0x2a, 0xb9, 0x6a, 0x3c, 0xea, 0x6f, 0xcb, 0xb8, 0x40, 0x6d, 0x25, 0x3e, 0x81, 0xdd, 0xb5, - 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xfb, 0x2d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0x83, 0xbf, 0x5d, - 0xe8, 0x58, 0x9e, 0x57, 0x45, 0x46, 0xfc, 0xec, 0x54, 0x45, 0x76, 0x83, 0x87, 0xcd, 0x19, 0xad, - 0x9e, 0x7a, 0x52, 0x17, 0x9c, 0xe3, 0xaa, 0x2c, 0x9d, 0x63, 0xd3, 0x08, 0xdd, 0xf3, 0x1a, 0x21, - 0x8d, 0xae, 0xe7, 0x61, 0x72, 0x22, 0x67, 0x5c, 0x96, 0x2d, 0xd4, 0x50, 0xec, 0x9b, 0xae, 0xc0, - 0xe7, 0xb8, 0xd6, 0x6b, 0xb4, 0x06, 0x4d, 0xe7, 0x50, 0x5d, 0x6e, 0x34, 0xa4, 0xb3, 0xe2, 0x7a, - 0x51, 0x48, 0xdc, 0x85, 0x8e, 0x69, 0x5f, 0x79, 0x75, 0x44, 0x3d, 0xe3, 0xca, 0x28, 0xd1, 0x36, - 0x14, 0x9f, 0x6e, 0xce, 0x25, 0xbf, 0xcd, 0x51, 0xf8, 0x6b, 0x99, 0x5b, 0x7a, 0xdc, 0x9c, 0x63, - 0x07, 0xd6, 0xa0, 0xf4, 0x81, 0x3f, 0xbe, 0x62, 0x3e, 0x5e, 0xa9, 0xd0, 0x1a, 0xa7, 0x77, 0xec, - 0x59, 0xe2, 0x77, 0xf8, 0x9b, 0xde, 0x3a, 0x73, 0x4a, 0x87, 0xf6, 0xcc, 0x39, 0xb0, 0x06, 0x99, - 0xdf, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0x15, 0xfc, 0x52, 0x83, 0x9d, 0xd1, 0x7c, 0x91, 0x66, - 0x85, 0x75, 0x0b, 0x47, 0xc9, 0x4c, 0xbe, 0xd0, 0xb7, 0x90, 0xc1, 0xf6, 0x41, 0xc5, 0xdd, 0x90, - 0x6e, 0x23, 0xdf, 0x3e, 0x0f, 0x15, 0xb0, 0x4e, 0xc0, 0x5b, 0x3b, 0x81, 0xeb, 0xd0, 0x56, 0xe5, - 0x46, 0xaa, 0x3a, 0xab, 0x8c, 0x40, 0x3d, 0x34, 0x96, 0x3c, 0xdc, 0x9b, 0x3c, 0xdc, 0x35, 0xa4, - 0xce, 0xa3, 0xcc, 0x58, 0xd9, 0x62, 0xa5, 0x25, 0x21, 0xfd, 0xd3, 0x68, 0x2e, 0xf3, 0x22, 0x9c, - 0x2f, 0xe8, 0x2a, 0xbb, 0x03, 0x17, 0x2d, 0x09, 0xdd, 0x62, 0x4e, 0xe2, 0x41, 0x26, 0xc3, 0x42, - 0xce, 0xee, 0x17, 0x7c, 0x82, 0x2e, 0x6e, 0x48, 0xc9, 0x8e, 0xd3, 0x32, 0x76, 0xa0, 0xec, 0xd6, - 0xa5, 0xc1, 0xaf, 0x35, 0x10, 0x8a, 0x33, 0xee, 0x7c, 0xff, 0x1f, 0x71, 0xe7, 0x13, 0xb4, 0x4e, - 0x43, 0xf3, 0x15, 0x1a, 0xae, 0x41, 0x83, 0xe3, 0xd1, 0x14, 0x54, 0x88, 0x1a, 0xa5, 0x69, 0xd3, - 0x8a, 0x3f, 0x07, 0x6d, 0x91, 0x08, 0xa0, 0x6b, 0xcd, 0x08, 0x2a, 0x70, 0xf2, 0xbd, 0x26, 0xdb, - 0x42, 0x22, 0x5c, 0x90, 0xc4, 0xce, 0x56, 0x12, 0x27, 0xd0, 0x7b, 0x9a, 0x85, 0x49, 0x1e, 0x87, - 0x85, 0xa4, 0xf0, 0x5f, 0x87, 0xc5, 0x2d, 0xaf, 0xda, 0xe0, 0x7d, 0xb8, 0xba, 0xe1, 0xd7, 0xb4, - 0x57, 0xa2, 0xd5, 0x65, 0x5a, 0x69, 0x19, 0x8c, 0xe1, 0xca, 0xca, 0x74, 0x34, 0x7c, 0xad, 0x08, - 0x5e, 0x75, 0xfa, 0x81, 0x95, 0x17, 0x3b, 0xad, 0xb6, 0xdf, 0x16, 0xeb, 0x21, 0xf8, 0xd5, 0xdd, - 0x53, 0x4f, 0xea, 0x2a, 0x82, 0x49, 0x24, 0x97, 0x64, 0x7f, 0x1c, 0xce, 0x65, 0x15, 0x04, 0xaf, - 0x49, 0xc6, 0xe3, 0xa5, 0xc6, 0x0f, 0x71, 0x5e, 0x07, 0x7f, 0x39, 0xd0, 0xdb, 0xe6, 0x84, 0xdf, - 0x4b, 0xb1, 0x0c, 0xd5, 0x40, 0x69, 0xa1, 0x02, 0xe2, 0x1e, 0xd4, 0xbf, 0x8b, 0xe4, 0x52, 0x0f, - 0x94, 0xc0, 0x7a, 0xeb, 0x9d, 0x11, 0x09, 0xaa, 0x0f, 0xa8, 0xbc, 0xee, 0x4f, 0x8b, 0x28, 0x4d, - 0xf4, 0xeb, 0x51, 0x21, 0xda, 0xe7, 0x30, 0x4e, 0xa7, 0xdf, 0x70, 0xdf, 0xf6, 0x50, 0x81, 0x2d, - 0xe5, 0x52, 0xbf, 0x60, 0xb9, 0x34, 0xb6, 0x96, 0xcb, 0x4f, 0x8e, 0xe6, 0xca, 0x9a, 0xf0, 0xff, - 0x79, 0x62, 0xea, 0x8e, 0xe9, 0xa7, 0x1a, 0xdf, 0x31, 0x5f, 0x3d, 0x53, 0xcc, 0x6b, 0x4c, 0x43, - 0x7a, 0x1a, 0xd1, 0x72, 0x12, 0xc6, 0xaa, 0x71, 0xb5, 0x71, 0x85, 0xcf, 0xbf, 0x99, 0x87, 0x97, - 0x7f, 0x7b, 0xb9, 0xe7, 0xfc, 0xfe, 0x72, 0xcf, 0xf9, 0xe3, 0xe5, 0x9e, 0xf3, 0xe3, 0x9f, 0x7b, - 0x6f, 0x3c, 0x6b, 0xf0, 0x1f, 0xda, 0x87, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x91, 0x85, - 0xb9, 0xb1, 0x0d, 0x00, 0x00, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b, + 0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c, + 0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55, + 0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5, + 0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10, + 0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c, + 0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda, + 0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6, + 0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82, + 0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39, + 0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1, + 0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c, + 0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c, + 0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a, + 0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f, + 0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc, + 0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b, + 0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4, + 0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c, + 0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce, + 0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87, + 0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19, + 0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72, + 0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a, + 0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65, + 0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07, + 0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54, + 0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b, + 0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33, + 0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9, + 0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c, + 0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39, + 0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce, + 0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d, + 0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90, + 0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a, + 0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98, + 0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c, + 0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a, + 0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42, + 0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81, + 0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5, + 0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9, + 0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c, + 0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41, + 0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06, + 0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11, + 0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd, + 0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad, + 0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b, + 0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99, + 0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4, + 0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2, + 0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0, + 0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3, + 0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35, + 0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a, + 0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b, + 0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51, + 0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71, + 0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1, + 0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0, + 0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea, + 0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88, + 0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0, + 0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda, + 0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2, + 0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9, + 0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb, + 0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3590,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x30 + } if len(m.ColumnIDs) > 0 { dAtA36 := make([]byte, len(m.ColumnIDs)*10) var j35 int @@ -4405,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9229,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index f1a1fa9fa..698581215 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -182,4 +182,5 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; + int64 IndexCreatedAt = 6; } From d8a417f657a5eb0b6538b91d3d9959693156c8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Fri, 29 May 2020 16:08:20 +0200 Subject: [PATCH 4/6] Move applyCreatedAt from mergeClusterStatus directly to ClusterStatus message, to avoid deadlocks --- api.go | 17 ++++++++--------- cluster.go | 3 --- pilosa.go | 2 +- server.go | 6 ++++++ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/api.go b/api.go index 54bb8f3be..08df4889c 100644 --- a/api.go +++ b/api.go @@ -181,15 +181,17 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + + createdAt := timestamp() index.mu.Lock() - index.createdAt = timestamp() + index.createdAt = createdAt index.mu.Unlock() // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ Index: indexName, - CreatedAt: index.CreatedAt(), + CreatedAt: createdAt, Meta: &options, }) if err != nil { @@ -274,15 +276,16 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + createdAt := timestamp() field.mu.Lock() - field.createdAt = timestamp() + field.createdAt = createdAt field.mu.Unlock() // Send the create field message to all nodes. err = api.server.SendSync(&CreateFieldMessage{ Index: indexName, Field: fieldName, - CreatedAt: field.CreatedAt(), + CreatedAt: createdAt, Meta: &fo, }) if err != nil { @@ -836,11 +839,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - if err := api.holder.applySchema(s); err != nil { - return errors.Wrap(err, "applying schema") - } - - return nil + return errors.Wrap(api.holder.applySchema(s), "applying schema") } // Views returns the views in the given field. diff --git a/cluster.go b/cluster.go index ced55e67a..2c50bceb9 100644 --- a/cluster.go +++ b/cluster.go @@ -2214,9 +2214,6 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { } } - if cs.Schema != nil { - c.holder.applyCreatedAt(cs.Schema.Indexes) - } c.unprotectedSetState(cs.State) c.markAsJoined() diff --git a/pilosa.go b/pilosa.go index 060141467..53733d9ce 100644 --- a/pilosa.go +++ b/pilosa.go @@ -205,7 +205,7 @@ func stringSlicesAreEqual(a, b []string) bool { } func timestamp() int64 { - return time.Now().UTC().UnixNano() + return time.Now().UnixNano() } // AddressWithDefaults converts addr into a valid address, diff --git a/server.go b/server.go index 8684a6061..9fbb30811 100644 --- a/server.go +++ b/server.go @@ -770,6 +770,12 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if !s.isCoordinator { + if obj.Schema != nil { + s.holder.applyCreatedAt(obj.Schema.Indexes) + } + } + case *ResizeInstruction: err := s.cluster.followResizeInstruction(obj) if err != nil { From 6a892102e42be5fbca7e3eb87ed58a0ebe88501d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 3 Jun 2020 13:25:37 +0200 Subject: [PATCH 5/6] Update api-reference.md --- docs/api-reference.md | 176 ++++++++++++++++++++++++++---------------- 1 file changed, 109 insertions(+), 67 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 68dae46b5..1c365bf83 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user ``` ``` response { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ + { + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } } + ], + "shardWidth": 1048576 } ``` @@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}' ``` ``` response -{"success":true} +{"success":true,"name":"user","createdAt":1591179042178854000} ``` ### Remove index @@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they exist must also contain the same number of items as rows and columns. The column IDs must all be in the shard specified in the request. +Some endpoints and data structures include a `CreatedAt` fields. +This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, +but to serve as a unique identifier for use in cache invalidation. + +The problem is that users of Pilosa (such as ingesters e.g. the IDK), +can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. +This is true except in cases where an index or field gets deleted and then recreated, +or if Pilosa is restored from a backup. +So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted), +and the ingester will know that it needs to drop its cache. + ``` message ImportRequest { - string Index = 1; - string Field = 2; - uint64 Shard = 3; - repeated uint64 RowIDs = 4; - repeated uint64 ColumnIDs = 5; - repeated string RowKeys = 7; - repeated string ColumnKeys = 8; - repeated int64 Timestamps = 6; + string Index = 1; + string Field = 2; + uint64 Shard = 3; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; + repeated int64 Timestamps = 6; + repeated string RowKeys = 7; + repeated string ColumnKeys = 8; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } ``` + ### Create field `POST /index//field/` @@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \ -d '{"options": {"type": "int", "min": -1000, "max":2000}}' ``` ``` response -{"success":true} +{"success":true,"name":"quantity","createdAt":1591180110914425000} ``` Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. @@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, curl localhost:10101/index/user/field/language -X POST ``` ``` response -{"success":true} +{"success":true,"name":"language","createdAt":1591180128294321000} ``` ``` request curl localhost:10101/index/repository/field/stats \ -X POST \ - -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"options":{"type": "int", "min": 0, "max": 1000000}}' ``` ``` response -{"success":true} +{"success":true,"name":"stats","createdAt":1591180737881627000} ``` ### Remove field @@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema ``` ``` response { - "indexes": [ + "indexes": [ + { + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - }, - { - "name": "language", - "options": { - "cacheSize": 50000, - "cacheType": "ranked", - "keys": false, - "type": "set" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true - } + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "language", + "createdAt": 1591180128294321000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "quantity", + "createdAt": 1591180110914425000, + "options": { + "type": "int", + "base": 0, + "bitDepth": 0, + "min": -1000, + "max": 2000, + "keys": false, + "foreignIndex": "" + } } - ] + ], + "shardWidth": 1048576 + } + ] } ``` @@ -304,7 +340,7 @@ Returns the version of the Pilosa server. curl -XGET localhost:10101/version ``` ``` response -{"version":"v0.6.0"} +{"version":"2.0.0-alpha.20-6-gb9d8d6b4"} ``` ### Get status @@ -318,19 +354,25 @@ curl -XGET localhost:10101/status ``` ```response { - "localID": "d3369125-29d8-4305-a351-b4474d14a542", - "nodes": [ - { - "id": "d3369125-29d8-4305-a351-b4474d14a542", - "isCoordinator": true, - "uri": { - "host": "localhost", - "port": 10101, - "scheme": "http" - } - } - ], - "state": "NORMAL" + "state": "NORMAL", + "nodes": [ + { + "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9", + "uri": { + "scheme": "http", + "host": "localhost", + "port": 10101 + }, + "grpc-uri": { + "scheme": "http", + "host": "localhost", + "port": 20101 + }, + "isCoordinator": true, + "state": "READY" + } + ], + "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9" } ``` From bb048da2414629f8e7b846e0cd96e642c85fc4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 3 Jun 2020 14:47:51 +0200 Subject: [PATCH 6/6] Update docs/api-reference.md Co-authored-by: Matthew Jaffee --- docs/api-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 1c365bf83..e8a2ac1a3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -159,7 +159,7 @@ Some endpoints and data structures include a `CreatedAt` fields. This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, but to serve as a unique identifier for use in cache invalidation. -The problem is that users of Pilosa (such as ingesters e.g. the IDK), +The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk)) can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. This is true except in cases where an index or field gets deleted and then recreated, or if Pilosa is restored from a backup.