diff --git a/api.go b/api.go index 210569f6a..08df4889c 100644 --- a/api.go +++ b/api.go @@ -181,11 +181,18 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + + createdAt := timestamp() + index.mu.Lock() + index.createdAt = createdAt + index.mu.Unlock() + // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ - Index: indexName, - Meta: &options, + Index: indexName, + CreatedAt: createdAt, + Meta: &options, }) if err != nil { return nil, errors.Wrap(err, "sending CreateIndex message") @@ -269,14 +276,18 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + createdAt := timestamp() + field.mu.Lock() + field.createdAt = createdAt + field.mu.Unlock() // Send the create field message to all nodes. - err = api.server.SendSync( - &CreateFieldMessage{ - Index: indexName, - Field: fieldName, - Meta: &fo, - }) + err = api.server.SendSync(&CreateFieldMessage{ + Index: indexName, + Field: fieldName, + CreatedAt: createdAt, + Meta: &fo, + }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) return nil, errors.Wrap(err, "sending CreateField message") @@ -413,14 +424,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "validating api method") } - nodes := api.cluster.shardNodes(indexName, shard) - - field := api.holder.Field(indexName, fieldName) - if field == nil { + index, field, err := api.indexField(indexName, fieldName, shard) + if index == nil || field == nil { return newNotFoundError(ErrFieldNotFound) } - errCh := make(chan error, len(nodes)) + if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return newPreconditionFailedError(err) + } + + nodes := api.cluster.shardNodes(indexName, shard) + errCh := make(chan error, len(nodes)) for _, node := range nodes { node := node if node.ID == api.server.nodeID { @@ -803,6 +817,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } + // set CreatedAt for indexes and fields (if empty), and then apply schema. + for _, index := range s.Indexes { + if index.CreatedAt == 0 { + index.CreatedAt = timestamp() + } + for _, field := range index.Fields { + if field.CreatedAt == 0 { + field.CreatedAt = timestamp() + } + } + } + if !remote { nodes := api.cluster.Nodes() for i, node := range nodes { @@ -813,7 +839,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - return api.holder.applySchema(s) + return errors.Wrap(api.holder.applySchema(s), "applying schema") } // Views returns the views in the given field. @@ -999,16 +1025,20 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "validating api method") } + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + // Set up import options. options, err := setUpImportOptions(opts...) if err != nil { return errors.Wrap(err, "setting up import options") } - - index, field, err := api.indexField(req.Index, req.Field, req.Shard) - if err != nil { - return errors.Wrap(err, "getting index and field") - } span.LogKV( "index", req.Index, "field", req.Field) @@ -1115,7 +1145,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating api method") } - if err := req.Validate(); err != nil { + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { return errors.Wrap(err, "validating import value request") } @@ -1125,7 +1160,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "setting up import options") } - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + index, field, err = api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } @@ -1266,6 +1301,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq return errors.Wrap(err, "validating shard ownership") } + if req.IndexCreatedAt != 0 { + if index.CreatedAt() != req.IndexCreatedAt { + return ErrPreconditionFailed + } + } + bulkAttrs := make(map[uint64]map[string]interface{}) for n := 0; n < len(req.ColumnIDs); n++ { bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} diff --git a/api_test.go b/api_test.go index b7f0fc06f..02e8a182f 100644 --- a/api_test.go +++ b/api_test.go @@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { m1 := c[1] t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() - index := "i" - field := "f" + indexName := "i" + fieldName := "f" attrKey := "k" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field) + _, err = m0.API.CreateField(ctx, indexName, fieldName) if err != nil { t.Fatalf("creating field: %v", err) } @@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { columnIDs0 = append(columnIDs0, uint64(n)) val0 := attrFun(uint64(n)) attrVals0 = append(attrVals0, val0) - setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field) - if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil { + setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName) + if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil { t.Fatal(err) } columnIDs1 = append(columnIDs1, uint64(n+ShardWidth)) val1 := attrFun(uint64(n + ShardWidth)) attrVals1 = append(attrVals1, val1) - setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field) - if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil { + setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName) + if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil { t.Fatal(err) } } // send shard0 to node1 req := &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs0, - AttrVals: attrVals0, - Shard: 0, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs0, + AttrVals: attrVals0, + Shard: 0, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { @@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { // send shard1 to node0 req = &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs1, - AttrVals: attrVals1, - Shard: 1, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs1, + AttrVals: attrVals1, + Shard: 1, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { @@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } // Query node0. - pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } } // Query node1. - pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -186,17 +188,24 @@ func TestAPI_Import(t *testing.T) { t.Run("RowIDColumnKey", func(t *testing.T) { ctx := context.Background() - index := "rick" - field := "f" + indexName := "rick" + fieldName := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } rowID := uint64(1) timestamp := int64(0) @@ -215,21 +224,23 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: index, - Field: field, - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), + Field: fieldName, + FieldCreatedAt: field.CreatedAt(), + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, } if err := m0.API.Import(ctx, req); err != nil { t.Fatal(err) } - pql := fmt.Sprintf("Row(%s=%d)", field, rowID) + pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { t.Fatal(err) } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %#v", keys) @@ -237,7 +248,7 @@ func TestAPI_Import(t *testing.T) { // Query node1. if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { return err } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { return fmt.Errorf("unexpected column keys: %#v", keys) diff --git a/cluster.go b/cluster.go index e979710d8..2c50bceb9 100644 --- a/cluster.go +++ b/cluster.go @@ -610,6 +610,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, + Schema: &Schema{Indexes: c.holder.Schema()}, } } @@ -1272,9 +1273,7 @@ func (c *cluster) listenForJoins() { // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. // We use a bool `setNormal` to indicate when at least one node has joined. var setNormal bool - for { - // Handle all pending joins before changing state back to NORMAL. select { case nodeAction := <-c.joiningLeavingNodes: @@ -2146,7 +2145,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name} + is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2155,6 +2154,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, }) } @@ -2450,6 +2450,7 @@ type ClusterStatus struct { ClusterID string State string Nodes []*Node + Schema *Schema } // ResizeInstruction contains the instruction provided to a node @@ -2491,7 +2492,7 @@ type translationResizeNode struct { // Schema contains information about indexes and their configuration. type Schema struct { - Indexes []*IndexInfo + Indexes []*IndexInfo `json:"indexes"` } func encodeTopology(topology *Topology) *internal.Topology { @@ -2529,8 +2530,9 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { - Index string - Meta *IndexOptions + Index string + CreatedAt int64 + Meta *IndexOptions } // DeleteIndexMessage is an internal message indicating index deletion. @@ -2540,9 +2542,10 @@ type DeleteIndexMessage struct { // CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { - Index string - Field string - Meta *FieldOptions + Index string + Field string + CreatedAt int64 + Meta *FieldOptions } // DeleteFieldMessage is an internal message indicating field deletion. @@ -2605,13 +2608,15 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { - Name string - Fields []*FieldStatus + Name string + CreatedAt int64 + Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string + CreatedAt int64 AvailableShards *roaring.Bitmap } diff --git a/docs/api-reference.md b/docs/api-reference.md index 68dae46b5..e8a2ac1a3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user ``` ``` response { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ + { + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } } + ], + "shardWidth": 1048576 } ``` @@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}' ``` ``` response -{"success":true} +{"success":true,"name":"user","createdAt":1591179042178854000} ``` ### Remove index @@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they exist must also contain the same number of items as rows and columns. The column IDs must all be in the shard specified in the request. +Some endpoints and data structures include a `CreatedAt` fields. +This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, +but to serve as a unique identifier for use in cache invalidation. + +The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk)) +can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. +This is true except in cases where an index or field gets deleted and then recreated, +or if Pilosa is restored from a backup. +So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted), +and the ingester will know that it needs to drop its cache. + ``` message ImportRequest { - string Index = 1; - string Field = 2; - uint64 Shard = 3; - repeated uint64 RowIDs = 4; - repeated uint64 ColumnIDs = 5; - repeated string RowKeys = 7; - repeated string ColumnKeys = 8; - repeated int64 Timestamps = 6; + string Index = 1; + string Field = 2; + uint64 Shard = 3; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; + repeated int64 Timestamps = 6; + repeated string RowKeys = 7; + repeated string ColumnKeys = 8; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } ``` + ### Create field `POST /index//field/` @@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \ -d '{"options": {"type": "int", "min": -1000, "max":2000}}' ``` ``` response -{"success":true} +{"success":true,"name":"quantity","createdAt":1591180110914425000} ``` Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. @@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, curl localhost:10101/index/user/field/language -X POST ``` ``` response -{"success":true} +{"success":true,"name":"language","createdAt":1591180128294321000} ``` ``` request curl localhost:10101/index/repository/field/stats \ -X POST \ - -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"options":{"type": "int", "min": 0, "max": 1000000}}' ``` ``` response -{"success":true} +{"success":true,"name":"stats","createdAt":1591180737881627000} ``` ### Remove field @@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema ``` ``` response { - "indexes": [ + "indexes": [ + { + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - }, - { - "name": "language", - "options": { - "cacheSize": 50000, - "cacheType": "ranked", - "keys": false, - "type": "set" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true - } + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "language", + "createdAt": 1591180128294321000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "quantity", + "createdAt": 1591180110914425000, + "options": { + "type": "int", + "base": 0, + "bitDepth": 0, + "min": -1000, + "max": 2000, + "keys": false, + "foreignIndex": "" + } } - ] + ], + "shardWidth": 1048576 + } + ] } ``` @@ -304,7 +340,7 @@ Returns the version of the Pilosa server. curl -XGET localhost:10101/version ``` ``` response -{"version":"v0.6.0"} +{"version":"2.0.0-alpha.20-6-gb9d8d6b4"} ``` ### Get status @@ -318,19 +354,25 @@ curl -XGET localhost:10101/status ``` ```response { - "localID": "d3369125-29d8-4305-a351-b4474d14a542", - "nodes": [ - { - "id": "d3369125-29d8-4305-a351-b4474d14a542", - "isCoordinator": true, - "uri": { - "host": "localhost", - "port": 10101, - "scheme": "http" - } - } - ], - "state": "NORMAL" + "state": "NORMAL", + "nodes": [ + { + "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9", + "uri": { + "scheme": "http", + "host": "localhost", + "port": 10101 + }, + "grpc-uri": { + "scheme": "http", + "host": "localhost", + "port": 20101 + }, + "isCoordinator": true, + "state": "READY" + } + ], + "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9" } ``` diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index a4508f652..82a31ce05 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { return &internal.ImportRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - RowIDs: m.RowIDs, - ColumnIDs: m.ColumnIDs, - RowKeys: m.RowKeys, - ColumnKeys: m.ColumnKeys, - Timestamps: m.Timestamps, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, } } func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { return &internal.ImportValueRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - ColumnIDs: m.ColumnIDs, - ColumnKeys: m.ColumnKeys, - Values: m.Values, - FloatValues: m.FloatValues, - StringValues: m.StringValues, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + FloatValues: m.FloatValues, + StringValues: m.StringValues, } } @@ -438,20 +442,23 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, } } func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest { return &internal.ImportColumnAttrsRequest{ - Index: m.Index, - Shard: m.Shard, - AttrKey: m.AttrKey, - AttrVals: m.AttrVals, - ColumnIDs: m.ColumnIDs, + Index: m.Index, + IndexCreatedAt: m.IndexCreatedAt, + Shard: m.Shard, + AttrKey: m.AttrKey, + AttrVals: m.AttrVals, + ColumnIDs: m.ColumnIDs, } } @@ -593,9 +600,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ - Name: idx.Name, - Options: s.encodeIndexMeta(&idx.Options), - Fields: s.encodeFieldInfos(idx.Fields), + Name: idx.Name, + CreatedAt: idx.CreatedAt, + Options: s.encodeIndexMeta(&idx.Options), + Fields: s.encodeFieldInfos(idx.Fields), } } @@ -609,9 +617,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ - Name: f.Name, - Meta: s.encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), + Name: f.Name, + CreatedAt: f.CreatedAt, + Meta: s.encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), } for _, viewinfo := range f.Views { @@ -672,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust State: m.State, ClusterID: m.ClusterID, Nodes: s.encodeNodes(m.Nodes), + Schema: s.encodeSchema(m.Schema), } } @@ -685,8 +695,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ - Index: m.Index, - Meta: s.encodeIndexMeta(m.Meta), + Index: m.Index, + CreatedAt: m.CreatedAt, + Meta: s.encodeIndexMeta(m.Meta), } } @@ -705,9 +716,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - Meta: s.encodeFieldOptions(m.Meta), + Index: m.Index, + Field: m.Field, + CreatedAt: m.CreatedAt, + Meta: s.encodeFieldOptions(m.Meta), } } @@ -786,8 +798,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ - Name: m.Name, - Fields: s.encodeFieldStatuses(m.Fields), + Name: m.Name, + CreatedAt: m.CreatedAt, + Fields: s.encodeFieldStatuses(m.Fields), } } @@ -802,6 +815,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, + CreatedAt: m.CreatedAt, AvailableShards: m.AvailableShards.Slice(), } } @@ -938,6 +952,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name + m.CreatedAt = idx.CreatedAt m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -953,6 +968,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name + m.CreatedAt = f.CreatedAt m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -992,6 +1008,8 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl m.ClusterID = cs.ClusterID m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) s.decodeNodes(cs.Nodes, m.Nodes) + m.Schema = &pilosa.Schema{} + s.decodeSchema(cs.Schema, m.Schema) } func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { @@ -1016,6 +1034,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1034,6 +1053,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1107,6 +1127,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name + m.CreatedAt = pb.CreatedAt m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1121,6 +1142,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name + m.CreatedAt = pb.CreatedAt m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } @@ -1149,6 +1171,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im m.RowKeys = pb.RowKeys m.ColumnKeys = pb.ColumnKeys m.Timestamps = pb.Timestamps + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { @@ -1160,6 +1184,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m m.Values = pb.Values m.FloatValues = pb.FloatValues m.StringValues = pb.StringValues + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { @@ -1171,10 +1197,13 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Action = pb.Action m.Block = int(pb.Block) m.Views = views + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { m.Index = pb.Index + m.IndexCreatedAt = pb.IndexCreatedAt m.Shard = pb.Shard m.AttrKey = pb.AttrKey m.AttrVals = pb.AttrVals diff --git a/field.go b/field.go index d38aa2ffb..84bdbec73 100644 --- a/field.go +++ b/field.go @@ -86,10 +86,11 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { - mu sync.RWMutex - path string - index string - name string + mu sync.RWMutex + createdAt int64 + path string + index string + name string viewMap map[string]*view @@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } +// CreatedAt is an timestamp for a specific version of field. +func (f *Field) CreatedAt() int64 { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.createdAt +} + // Index returns the index name the field was initialized with. func (f *Field) Index() string { return f.index } @@ -1971,9 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/gossip/gossip.go b/gossip/gossip.go index 84f914975..975c57a19 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name} + is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} + for _, f := range idx.Fields { availableShards := roaring.NewBitmap() if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() } - is.Fields = append(is.Fields, &pilosa.FieldStatus{ + + fs := &pilosa.FieldStatus{ Name: f.Name, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, - }) + } + is.Fields = append(is.Fields, fs) } m.Indexes = append(m.Indexes, is) } diff --git a/handler.go b/handler.go index 3c40d20ad..17156245b 100644 --- a/handler.go +++ b/handler.go @@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. type ImportValueRequest struct { - Index string - Field string + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 @@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) { // Validate ensures that the payload of the request is valid. func (ivr *ImportValueRequest) Validate() error { + return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt) +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if ivr.Index == "" || ivr.Field == "" { return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field) } @@ -160,30 +167,48 @@ func (ivr *ImportValueRequest) Validate() error { if valueSetCount > 1 { return errors.Errorf("must pass ints, floats, or strings but not multiple") } + if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 { + if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } return nil } // ImportColumnAttrsRequest describes the import request structure // for a ColumnAttr import type ImportColumnAttrsRequest struct { - AttrKey string - ColumnIDs []uint64 - AttrVals []string - Shard int64 - Index string + AttrKey string + ColumnIDs []uint64 + AttrVals []string + Shard int64 + Index string + IndexCreatedAt int64 } // ImportRequest describes the import request structure // for an import. type ImportRequest struct { - Index string - Field string - Shard uint64 - RowIDs []uint64 - ColumnIDs []uint64 - RowKeys []string - ColumnKeys []string - Timestamps []int64 + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 { + if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } const ( @@ -195,10 +220,22 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 { + if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } // ImportResponse is the structured response of an import. diff --git a/holder.go b/holder.go index ad52ee082..70501a4b1 100644 --- a/holder.go +++ b/holder.go @@ -234,7 +234,13 @@ func (h *Holder) Open() error { return errors.Wrap(err, "opening index") } - if err := index.Open(); err != nil { + if h.isCoordinator() { + index.createdAt = timestamp() + err = index.OpenWithTimestamp() + } else { + err = index.Open() + } + if err != nil { if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue @@ -380,11 +386,16 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ - Name: index.Name(), - Options: index.Options(), + Name: index.Name(), + CreatedAt: index.CreatedAt(), + Options: index.Options(), } for _, field := range index.Fields() { - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), + } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } @@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), + } di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo { // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. - for _, index := range schema.Indexes { - idx, err := h.CreateIndexIfNotExists(index.Name, index.Options) + for _, i := range schema.Indexes { + idx, err := h.CreateIndexIfNotExists(i.Name, i.Options) if err != nil { return errors.Wrap(err, "creating index") } + if i.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = i.CreatedAt + idx.mu.Unlock() + } + // Create fields that don't exist. - for _, f := range index.Fields { - field, err := idx.createFieldIfNotExists(f.Name, &f.Options) + for _, f := range i.Fields { + fld, err := idx.createFieldIfNotExists(f.Name, &f.Options) if err != nil { return errors.Wrap(err, "creating field") } + if f.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = f.CreatedAt + fld.mu.Unlock() + } + // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v.Name) + _, err := fld.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -447,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } +func (h *Holder) applyCreatedAt(indexes []*IndexInfo) { + for _, ii := range indexes { + idx := h.Index(ii.Name) + if idx == nil { + continue + } + if ii.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = ii.CreatedAt + idx.mu.Unlock() + } + + for _, fi := range ii.Fields { + fld := idx.Field(fi.Name) + if fld == nil { + continue + } + if fi.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = fi.CreatedAt + fld.mu.Unlock() + } + } + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } @@ -652,6 +706,13 @@ func (h *Holder) recalculateCaches() { } } +func (h *Holder) isCoordinator() bool { + if s, ok := h.broadcaster.(*Server); ok { + return s.isCoordinator + } + return false +} + // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { oldLimit := &syscall.Rlimit{} diff --git a/http/handler.go b/http/handler.go index 5b9eaf405..12773f3ee 100644 --- a/http/handler.go +++ b/http/handler.go @@ -15,13 +15,13 @@ package http import ( + "bytes" "context" "crypto/tls" "encoding/json" "expvar" "fmt" "io" - "io/ioutil" "math" "net" "net/http" @@ -401,9 +401,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { - h *Handler - Success bool `json:"success"` - Error *Error `json:"error,omitempty"` + h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *Error `json:"error,omitempty"` } // check determines success or failure based on the error. @@ -504,7 +506,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -773,7 +775,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: indexName} // Decode request. req := postIndexRequest{ @@ -787,8 +789,15 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) + index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { + resp.CreatedAt = index.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + if index, _ = h.api.Index(r.Context(), indexName); index != nil { + resp.CreatedAt = index.CreatedAt() + } + } resp.write(w, err) } @@ -853,7 +862,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: fieldName} // Decode request. var req postFieldRequest @@ -925,11 +934,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex)) } - _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) if _, ok := err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } + if field != nil { + resp.CreatedAt = field.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { + resp.CreatedAt = field.CreatedAt() + } + } resp.write(w, err) } @@ -1240,7 +1256,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error // readProtobufQueryRequest parses query parameters in protobuf from r. func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { // Slurp the body. - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1258,7 +1274,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er q := r.URL.Query() // Parse query string. - buf, err := ioutil.ReadAll(r.Body) + buf, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1319,95 +1335,14 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse return json.NewEncoder(w).Encode(resp) } -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. +func validateProtobufHeader(r *http.Request) (error string, code int) { if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return + return "Unsupported media type", http.StatusUnsupportedMediaType } - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // If the clear flag is true, treat the import as clear bits. - q := r.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), - } - - // Get index and field type to determine how to handle the - // import data. - field, err := h.api.Field(r.Context(), indexName, fieldName) - if err != nil { - switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: - fallthrough - case pilosa.ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { - // Field type: Int - // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.Import(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } - - // Write response. - _, err = w.Write(importOk) - if err != nil { - h.logger.Printf("writing import response: %v", err) + if r.Header.Get("Accept") != "application/x-protobuf" { + return "Not acceptable", http.StatusNotAcceptable } + return } // handleGetExport handles /export requests. @@ -1885,6 +1820,95 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // If the clear flag is true, treat the import as clear bits. + q := r.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + + opts := []pilosa.ImportOption{ + pilosa.OptImportOptionsClear(doClear), + pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + } + + // Read entire body. + body, err := readBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Unmarshal request based on field type. + if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { + // Field type: Int + // Marshal into request object. + req := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + req := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.Import(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + // Write response. + _, err = w.Write(importOk) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } +} + // handlePostImportColumnAttrs func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. @@ -1898,7 +1922,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req opts := []pilosa.ImportOption{} - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1911,7 +1935,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1922,16 +1951,16 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } } -// handlPostRoaringImport +// handlePostImportRoaring func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) return } + + // Get index and field type to determine how to handle the + // import data. indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] @@ -1946,7 +1975,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Read entire body. span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() if err != nil { @@ -1976,6 +2005,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) + } else if _, ok := err.(pilosa.NotFoundError); ok { + w.WriteHeader(http.StatusNotFound) + } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) } @@ -2040,3 +2073,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) h.logger.Printf("writing translate keys response: %v", err) } } + +// Read entire request body. +func readBody(r *http.Request) ([]byte, error) { + var contentLength int64 = bytes.MinRead + if r.ContentLength > 0 { + contentLength = r.ContentLength + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength)) + if _, err := buf.ReadFrom(r.Body); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/index.go b/index.go index a5f143bfc..481d22cf4 100644 --- a/index.go +++ b/index.go @@ -36,10 +36,11 @@ import ( // Index represents a container for fields. type Index struct { - mu sync.RWMutex - path string - name string - keys bool // use string keys + mu sync.RWMutex + createdAt int64 + path string + name string + keys bool // use string keys // Existence tracking. trackExistence bool @@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } +// CreatedAt is an timestamp for a specific version of an index. +func (i *Index) CreatedAt() int64 { + i.mu.RLock() + defer i.mu.RUnlock() + return i.createdAt +} + // Name returns name of the index. func (i *Index) Name() string { return i.name } @@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open() (err error) { +func (i *Index) Open() error { return i.open(false) } + +// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. +func (i *Index) OpenWithTimestamp() error { return i.open(true) } + +func (i *Index) open(withTimestamp bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -154,7 +167,7 @@ func (i *Index) Open() (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(); err != nil { + if err := i.openFields(withTimestamp); err != nil { return errors.Wrap(err, "opening fields") } @@ -197,7 +210,7 @@ func (i *Index) Open() (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields() error { +func (i *Index) openFields(withTimestamp bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -229,6 +242,9 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if withTimestamp { + fld.createdAt = timestamp() + } mu.Unlock() if err != nil { return errors.Wrapf(ErrName, "'%s'", fi.Name()) @@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index 38eb7af3d..6dd3eb9bc 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } +func (m *CreateIndexMessage) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } +func (m *CreateFieldMessage) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -853,6 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -912,6 +929,13 @@ func (m *Field) GetViews() []string { return nil } +func (m *Field) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type Schema struct { Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1008,6 +1033,13 @@ func (m *Index) GetName() string { return "" } +func (m *Index) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + func (m *Index) GetOptions() *IndexMeta { if m != nil { return m.Options @@ -1340,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1392,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } +func (m *IndexStatus) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1447,10 +1488,18 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } +func (m *FieldStatus) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1510,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } +func (m *ClusterStatus) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -2421,96 +2477,99 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1418 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0x1b, 0xc5, - 0x13, 0xff, 0xaf, 0x56, 0xb6, 0xa4, 0x96, 0xe5, 0xc8, 0x93, 0xc4, 0xd9, 0xf8, 0x4f, 0x19, 0x31, - 0xa4, 0x88, 0x48, 0x15, 0x26, 0x95, 0x50, 0xc5, 0x67, 0xaa, 0x12, 0x5b, 0x4e, 0x10, 0xc1, 0x8e, - 0x33, 0x72, 0x72, 0xe3, 0x30, 0x5e, 0x4d, 0xc5, 0x5b, 0x5e, 0xed, 0x8a, 0xdd, 0x59, 0x47, 0xce, - 0x81, 0x2b, 0x54, 0xf1, 0x02, 0x1c, 0x38, 0xf0, 0x1e, 0xbc, 0x00, 0x47, 0x1e, 0x81, 0x0a, 0x4f, - 0xc1, 0x8d, 0x9a, 0x9e, 0x99, 0xdd, 0x95, 0x2c, 0xe3, 0x90, 0x70, 0xdb, 0xfe, 0xf5, 0x77, 0x4f, - 0x77, 0xcf, 0x2c, 0xb4, 0xc6, 0x49, 0x70, 0xcc, 0xa5, 0xd8, 0x18, 0x27, 0xb1, 0x8c, 0x49, 0x3d, - 0x88, 0xa4, 0x48, 0x22, 0x1e, 0xae, 0x2d, 0x8d, 0xb3, 0x83, 0x30, 0xf0, 0x35, 0x4e, 0x1f, 0x40, - 0xa3, 0x1f, 0x0d, 0xc5, 0x64, 0x47, 0x48, 0x4e, 0x08, 0x54, 0x1f, 0x8a, 0x93, 0xd4, 0x73, 0x3b, - 0x4e, 0xb7, 0xce, 0xf0, 0x9b, 0xbc, 0x07, 0xcb, 0xfb, 0x09, 0xf7, 0x8f, 0xb6, 0x27, 0x41, 0x2a, - 0x45, 0xe4, 0x0b, 0xaf, 0x8a, 0xdc, 0x19, 0x94, 0xfe, 0xe2, 0xc2, 0xd2, 0xfd, 0x40, 0x84, 0xc3, - 0x47, 0x63, 0x19, 0xc4, 0x51, 0xaa, 0x8c, 0xed, 0x9f, 0x8c, 0x85, 0x57, 0xef, 0x38, 0xdd, 0x06, - 0xc3, 0x6f, 0xf2, 0x16, 0x34, 0xb6, 0xb8, 0x7f, 0x28, 0x90, 0xe1, 0x22, 0xa3, 0x00, 0x72, 0xee, - 0x20, 0x78, 0xa1, 0xbd, 0xb4, 0x58, 0x01, 0x90, 0x0e, 0x34, 0xf7, 0x83, 0x91, 0x78, 0x9c, 0xf1, - 0x48, 0x66, 0x23, 0x6f, 0x01, 0xb5, 0xcb, 0x10, 0x59, 0x85, 0xc5, 0x47, 0xe1, 0x70, 0x27, 0x88, - 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, 0x50, 0x16, 0xe7, 0x13, 0x0f, 0x0a, 0x9c, 0x4f, 0xf2, 0x74, - 0x9b, 0xd3, 0xe9, 0xee, 0xc6, 0x03, 0xc9, 0xa3, 0x21, 0x4f, 0x86, 0x4f, 0x03, 0xf1, 0xdc, 0x5b, - 0xd2, 0xe9, 0x4e, 0xa3, 0x4a, 0x77, 0x93, 0xa7, 0xc2, 0x6b, 0xa1, 0x45, 0xfc, 0x26, 0x6b, 0x50, - 0xdf, 0x0c, 0x64, 0x4f, 0x8c, 0xe5, 0xa1, 0xb7, 0xdc, 0x71, 0xba, 0x55, 0x96, 0xd3, 0xe4, 0x12, - 0x2c, 0x0c, 0x7c, 0x1e, 0x0a, 0xef, 0x02, 0x2a, 0x68, 0x82, 0x50, 0x58, 0xba, 0x1f, 0x27, 0x22, - 0x78, 0x16, 0xe1, 0x21, 0x78, 0x6d, 0x4c, 0x6a, 0x0a, 0x23, 0xef, 0x82, 0xab, 0x52, 0x5a, 0xe9, - 0x38, 0xdd, 0xe6, 0xad, 0x95, 0x0d, 0x7b, 0x8e, 0x1b, 0x3d, 0xe1, 0x07, 0x23, 0x1e, 0x32, 0xc5, - 0x45, 0x21, 0x3e, 0xf1, 0xc8, 0xd9, 0x42, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x38, 0x91, - 0x4c, 0xa4, 0xe3, 0x38, 0x4a, 0x05, 0x69, 0x83, 0xbb, 0x9d, 0x24, 0x9e, 0x83, 0x6e, 0xd5, 0x27, - 0xfd, 0x0e, 0xda, 0x9b, 0x61, 0xec, 0x1f, 0xf5, 0xb8, 0xe4, 0x4c, 0x7c, 0x9b, 0x89, 0x54, 0xaa, - 0xd8, 0x75, 0x78, 0x5a, 0x4e, 0x13, 0x0a, 0xc5, 0xf3, 0xf6, 0x2a, 0x1a, 0x45, 0x42, 0xd5, 0x05, - 0xab, 0xa6, 0x8f, 0x07, 0xbf, 0x31, 0xf7, 0x43, 0x9e, 0x0c, 0xf1, 0x4c, 0xab, 0x4c, 0x13, 0x0a, - 0x45, 0x4f, 0xd8, 0x07, 0x55, 0xa6, 0x09, 0xda, 0x87, 0x95, 0x92, 0x7f, 0x13, 0xe6, 0x2a, 0x2c, - 0xb2, 0xf8, 0x79, 0xbf, 0x97, 0x7a, 0x4e, 0xc7, 0xed, 0x56, 0x99, 0xa1, 0xb0, 0x61, 0xe2, 0x30, - 0x1b, 0x45, 0x8a, 0x55, 0x41, 0x56, 0x01, 0xd0, 0xab, 0xb0, 0x80, 0xdd, 0xa3, 0xb2, 0x2c, 0x74, - 0xd5, 0x27, 0xfd, 0xde, 0x81, 0xc6, 0x0e, 0x9f, 0x60, 0x20, 0x29, 0xb9, 0x03, 0x75, 0x7b, 0xb6, - 0x28, 0xd4, 0xbc, 0xf5, 0x4e, 0x51, 0xc1, 0x5c, 0x6c, 0xc3, 0xca, 0x6c, 0x47, 0x32, 0x39, 0x61, - 0xb9, 0xca, 0xda, 0xe7, 0xd0, 0x9a, 0x62, 0x29, 0x7f, 0x47, 0xe2, 0xc4, 0x56, 0xf5, 0x48, 0x9c, - 0xa8, 0x5c, 0x8f, 0x79, 0x98, 0x09, 0xac, 0x55, 0x95, 0x69, 0xe2, 0xb3, 0xca, 0x27, 0x0e, 0x7d, - 0x0a, 0x64, 0x2b, 0x11, 0x5c, 0x0a, 0x74, 0xb2, 0x23, 0xd2, 0x94, 0x3f, 0x13, 0xe7, 0x55, 0xdc, - 0x2d, 0x57, 0x3c, 0xaf, 0x6e, 0xa5, 0x54, 0x5d, 0x7a, 0x03, 0x48, 0x4f, 0x84, 0x42, 0x0a, 0x33, - 0xdd, 0xff, 0x60, 0x97, 0x0e, 0x6c, 0x0c, 0xe7, 0xcb, 0x92, 0xeb, 0x50, 0x55, 0xab, 0x02, 0x9d, - 0x35, 0x6f, 0x5d, 0x2c, 0xea, 0x94, 0x6f, 0x11, 0x86, 0x02, 0x34, 0xb4, 0x46, 0x31, 0xca, 0x57, - 0x4c, 0x6c, 0xaa, 0x95, 0x6e, 0x18, 0x57, 0x2e, 0xba, 0x5a, 0x2d, 0x5c, 0x95, 0xd7, 0x8c, 0xf1, - 0x76, 0xd7, 0xa6, 0xfb, 0xba, 0xde, 0xa8, 0x0f, 0xff, 0xd7, 0x16, 0xee, 0x1d, 0xf3, 0x20, 0xe4, - 0x07, 0xe1, 0xbf, 0x3a, 0x91, 0xa9, 0xc0, 0x3d, 0xa8, 0xa1, 0x6e, 0xbf, 0x67, 0x7a, 0xdb, 0x92, - 0xf4, 0x1b, 0x28, 0xc6, 0x64, 0x97, 0x8f, 0x84, 0xb1, 0x86, 0xdf, 0x79, 0xbe, 0x95, 0xf3, 0xf3, - 0x55, 0x8e, 0xd5, 0x68, 0xa9, 0x55, 0xed, 0x2a, 0xc7, 0x48, 0xd0, 0xdb, 0xb0, 0x38, 0xf0, 0x0f, - 0xc5, 0x88, 0x93, 0xf7, 0xa1, 0x86, 0x11, 0x8a, 0xd4, 0x74, 0xf4, 0x85, 0x99, 0x93, 0x62, 0x96, - 0x4f, 0x53, 0x93, 0xd9, 0xdc, 0x98, 0x3e, 0x80, 0x9a, 0x71, 0x8c, 0x13, 0x7d, 0xc6, 0x89, 0x5b, - 0x19, 0x72, 0x1d, 0x16, 0x31, 0xd8, 0xd4, 0xab, 0xce, 0x7a, 0x45, 0x9c, 0x19, 0x36, 0xdd, 0x06, - 0xf7, 0x09, 0xeb, 0xab, 0xc1, 0xc6, 0x80, 0xad, 0x53, 0x43, 0xa9, 0x50, 0xbe, 0x8c, 0x53, 0x69, - 0xca, 0x8a, 0xdf, 0x0a, 0xdb, 0x8b, 0x13, 0x89, 0x25, 0x6d, 0x31, 0xfc, 0xa6, 0x3f, 0x3b, 0x50, - 0xdd, 0x8d, 0x87, 0x82, 0x2c, 0x43, 0xa5, 0xdf, 0x33, 0x46, 0x2a, 0xfd, 0x1e, 0x79, 0x1b, 0xed, - 0x9b, 0x52, 0xb6, 0x8a, 0x28, 0x9e, 0xb0, 0x3e, 0x43, 0xcf, 0xd7, 0xa0, 0xd5, 0x4f, 0xb7, 0xe2, - 0x38, 0x19, 0x06, 0x11, 0x97, 0x71, 0x62, 0xee, 0xbc, 0x69, 0x10, 0x67, 0x4b, 0x72, 0xa9, 0x6f, - 0xa3, 0x06, 0xd3, 0x04, 0xb9, 0x0e, 0xb5, 0x07, 0x6c, 0x6f, 0x4b, 0x39, 0x58, 0x98, 0xe7, 0xc0, - 0x72, 0xe9, 0x5d, 0x68, 0xab, 0xe8, 0x50, 0xcb, 0x36, 0xd2, 0x2a, 0x2c, 0x2a, 0x2c, 0x8f, 0xd6, - 0x50, 0x85, 0xab, 0x4a, 0xc9, 0x15, 0xfd, 0x5a, 0x5b, 0xd8, 0x3e, 0x16, 0x91, 0x2c, 0xb5, 0x22, - 0xd2, 0x68, 0xa0, 0xc5, 0x34, 0x41, 0xa8, 0xae, 0x84, 0x49, 0x79, 0xb9, 0x88, 0x48, 0xa1, 0x0c, - 0x79, 0xf4, 0x47, 0x07, 0xc0, 0x06, 0x94, 0xa5, 0xb9, 0x8a, 0x73, 0xb6, 0x0a, 0xe9, 0xda, 0x96, - 0x32, 0x63, 0xd8, 0x2e, 0xa4, 0x34, 0xce, 0x6c, 0xcb, 0x7d, 0x58, 0xb4, 0x9c, 0x3e, 0xfc, 0xcb, - 0x33, 0xad, 0xa2, 0xbd, 0x16, 0x8d, 0xb7, 0x07, 0xcd, 0x12, 0x7e, 0x46, 0xfb, 0xd9, 0x7e, 0xaa, - 0xcc, 0x9a, 0x44, 0xdc, 0x98, 0xb4, 0x5d, 0xf5, 0x10, 0x9a, 0x25, 0x78, 0xae, 0xc5, 0x2e, 0x5c, - 0x98, 0x1e, 0x70, 0x7b, 0x71, 0xcc, 0xc2, 0x34, 0x80, 0xd6, 0x56, 0x98, 0xa5, 0x52, 0x24, 0xc6, - 0x9c, 0xba, 0x6d, 0x34, 0x90, 0x1f, 0x5e, 0x01, 0xcc, 0x3f, 0x3f, 0x72, 0x0d, 0x16, 0x54, 0x19, - 0xf5, 0x9c, 0x9e, 0xae, 0xb1, 0x66, 0xd2, 0xa7, 0x50, 0xdf, 0x1c, 0xf4, 0x1f, 0x24, 0x71, 0x36, - 0x9e, 0x1b, 0xb4, 0x7d, 0x4a, 0x55, 0x4a, 0x4f, 0xa9, 0xb6, 0x7e, 0x16, 0xb8, 0xf8, 0x9c, 0xc0, - 0x37, 0x40, 0x5b, 0xbf, 0x01, 0xaa, 0x06, 0xe1, 0x6a, 0xb1, 0xaf, 0xe8, 0x1d, 0xac, 0xd6, 0xc3, - 0xeb, 0x6c, 0x32, 0x7b, 0x9b, 0xbb, 0xc5, 0x6d, 0xae, 0x8c, 0xea, 0x45, 0xf9, 0x5f, 0x1a, 0xfd, - 0xab, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x0b, 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x27, 0x4a, - 0xff, 0xab, 0xf8, 0xc0, 0x54, 0xdb, 0x65, 0x9a, 0x78, 0x95, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, - 0xdc, 0xa7, 0x45, 0xcb, 0x22, 0xe4, 0x26, 0xd4, 0x06, 0x71, 0x96, 0xf8, 0x79, 0xfb, 0x96, 0x16, - 0xb0, 0x8e, 0x4c, 0xb3, 0x99, 0x15, 0x23, 0x8f, 0x81, 0xec, 0x27, 0x3c, 0x4a, 0x43, 0xae, 0x82, - 0xb5, 0xca, 0xf5, 0xd9, 0x07, 0x44, 0x49, 0x66, 0xca, 0xce, 0x1c, 0x65, 0xf2, 0x51, 0x79, 0x3e, - 0xbd, 0x1a, 0x46, 0x7d, 0x69, 0x3a, 0x6a, 0xd3, 0xf2, 0xe5, 0x39, 0xbe, 0x33, 0xd3, 0xa9, 0xde, - 0x22, 0x2a, 0x5e, 0x29, 0x14, 0xa7, 0xd8, 0x6c, 0x5a, 0x9a, 0xfe, 0xe0, 0xc0, 0x52, 0x39, 0xb2, - 0x57, 0xda, 0x0b, 0xf9, 0x81, 0x57, 0xce, 0x7f, 0xa1, 0xd8, 0x03, 0xaf, 0xce, 0x7b, 0x13, 0x2e, - 0x94, 0x5f, 0x2d, 0x19, 0x5c, 0x39, 0xa3, 0x5c, 0x6f, 0x10, 0x54, 0x07, 0x9a, 0x7b, 0x3c, 0x91, - 0x81, 0x32, 0x69, 0xae, 0xe4, 0x05, 0x56, 0x86, 0xe8, 0x11, 0x5c, 0x3d, 0xd5, 0x7c, 0x5b, 0xf1, - 0x68, 0xac, 0xba, 0xfc, 0x0d, 0x9a, 0x50, 0x2d, 0xea, 0x24, 0x31, 0xed, 0xd7, 0x60, 0x9a, 0xa0, - 0x9f, 0xc2, 0xe5, 0x81, 0x90, 0xa5, 0xd6, 0xb3, 0x33, 0xd4, 0x01, 0x77, 0x57, 0x3c, 0x3f, 0x23, - 0x41, 0xc5, 0xa2, 0x5f, 0x80, 0xf7, 0x64, 0x3c, 0xe4, 0x52, 0xbc, 0x96, 0xf6, 0x26, 0xd4, 0xf7, - 0xe3, 0x71, 0x1c, 0xc6, 0xcf, 0x4e, 0xce, 0xd9, 0x65, 0x1e, 0xd4, 0xf4, 0xad, 0xa4, 0x97, 0x63, - 0x83, 0x59, 0x92, 0x5e, 0x54, 0x63, 0xea, 0xf3, 0xd0, 0xcf, 0x42, 0x15, 0x86, 0x7a, 0x5e, 0xa7, - 0x54, 0x98, 0x41, 0xe0, 0x58, 0xb8, 0xd2, 0x45, 0x77, 0x0f, 0x01, 0x7b, 0xd1, 0x69, 0x8a, 0x7c, - 0x0c, 0xcd, 0x92, 0xb4, 0x29, 0xe0, 0xe5, 0x99, 0x79, 0xd1, 0x4c, 0x56, 0x96, 0xa4, 0xbf, 0x3a, - 0x53, 0x9a, 0xa7, 0xee, 0x7c, 0xe3, 0xf0, 0x58, 0x1f, 0x4a, 0x9d, 0x19, 0x4a, 0xe5, 0xba, 0x3d, - 0xf1, 0xc3, 0x2c, 0x55, 0x2c, 0x7d, 0xcd, 0x17, 0x80, 0xca, 0x55, 0xfd, 0x43, 0xc6, 0x99, 0x34, - 0x9b, 0xd3, 0x92, 0xea, 0x77, 0xae, 0x27, 0xf8, 0x30, 0x0c, 0x22, 0x81, 0x5d, 0xea, 0xb2, 0x9c, - 0x26, 0x37, 0xf5, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x37, 0x7c, 0x94, 0xd0, 0x37, 0x41, 0x4a, 0x09, - 0xb4, 0x67, 0x59, 0x9b, 0xed, 0xdf, 0x5e, 0xae, 0x3b, 0xbf, 0xbf, 0x5c, 0x77, 0xfe, 0x78, 0xb9, - 0xee, 0xfc, 0xf4, 0xe7, 0xfa, 0xff, 0x0e, 0x16, 0xf1, 0xaf, 0xfc, 0xf6, 0xdf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x8a, 0x80, 0x2a, 0x36, 0xbe, 0x0f, 0x00, 0x00, + // 1458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45, + 0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf, + 0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0, + 0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa, + 0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0, + 0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97, + 0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75, + 0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e, + 0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec, + 0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17, + 0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8, + 0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba, + 0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91, + 0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47, + 0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee, + 0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27, + 0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78, + 0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93, + 0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3, + 0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17, + 0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29, + 0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb, + 0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38, + 0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0, + 0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45, + 0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21, + 0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, + 0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, + 0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, + 0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, + 0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37, + 0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2, + 0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87, + 0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4, + 0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf, + 0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89, + 0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, + 0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, + 0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, + 0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, + 0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf, + 0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3, + 0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce, + 0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e, + 0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce, + 0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15, + 0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0, + 0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae, + 0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79, + 0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64, + 0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47, + 0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78, + 0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82, + 0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68, + 0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5, + 0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5, + 0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b, + 0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce, + 0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2, + 0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44, + 0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae, + 0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27, + 0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d, + 0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02, + 0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56, + 0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0, + 0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91, + 0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07, + 0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38, + 0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c, + 0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04, + 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92, + 0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2, + 0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59, + 0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98, + 0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01, + 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e, + 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10, + 0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19, + 0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2, + 0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d, + 0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe, + 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3, + 0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a, + 0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9, + 0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c, + 0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f, + 0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b, + 0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb, + 0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05, + 0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3043,6 +3102,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3089,6 +3153,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x20 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3229,6 +3298,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x20 + } if len(m.Views) > 0 { for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Views[iNdEx]) @@ -3351,6 +3425,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x10 + } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) @@ -3656,6 +3735,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if len(m.Fields) > 0 { for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { { @@ -3704,6 +3788,11 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if len(m.AvailableShards) > 0 { dAtA19 := make([]byte, len(m.AvailableShards)*10) var j18 int @@ -3756,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Schema != nil { + { + size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if len(m.Nodes) > 0 { for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { { @@ -4759,6 +4860,9 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4783,6 +4887,9 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4852,6 +4959,9 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4886,6 +4996,9 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -5037,6 +5150,9 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5060,6 +5176,9 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5086,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7021,6 +7144,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7175,6 +7317,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7584,6 +7745,25 @@ func (m *Field) Unmarshal(dAtA []byte) error { } m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7758,6 +7938,25 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -8682,6 +8881,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8844,6 +9062,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType) } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8996,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 849fff04a..d83a8d2c0 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,12 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; + int64 CreatedAt = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; + int64 CreatedAt = 4; } message DeleteFieldMessage { @@ -87,6 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; + int64 CreatedAt = 4; } message Schema { @@ -95,6 +98,7 @@ message Schema { message Index { string Name = 1; + int64 CreatedAt = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -132,17 +136,20 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; + int64 CreatedAt = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; + int64 CreatedAt = 3; } message ClusterStatus { string ClusterID = 1; string State = 2; repeated Node Nodes = 3; + Schema Schema = 4; } message BSIGroup { diff --git a/internal/public.pb.go b/internal/public.pb.go index bf04b2d90..9f71b0368 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1183,6 +1183,8 @@ type ImportRequest struct { RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 { return nil } +func (m *ImportRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1286,6 +1302,8 @@ type ImportValueRequest struct { Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string { return nil } +func (m *ImportValueRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportValueRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct { Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"` Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1726,12 +1760,27 @@ func (m *ImportRoaringRequest) GetBlock() uint64 { return 0 } +func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"` AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1805,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 { return nil } +func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") @@ -1837,83 +1893,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1207 bytes of a gzipped FileDescriptorProto + // 1258 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b, - 0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b, - 0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00, - 0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d, - 0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa, - 0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a, - 0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b, - 0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e, - 0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf, - 0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92, - 0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a, - 0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85, - 0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23, - 0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42, - 0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3, - 0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5, - 0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87, - 0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54, - 0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b, - 0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87, - 0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a, - 0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e, - 0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36, - 0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75, - 0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72, - 0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae, - 0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c, - 0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde, - 0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68, - 0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2, - 0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c, - 0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47, - 0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0, - 0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3, - 0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e, - 0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17, - 0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97, - 0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61, - 0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb, - 0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29, - 0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef, - 0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c, - 0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30, - 0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea, - 0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0, - 0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5, - 0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c, - 0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d, - 0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b, - 0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43, - 0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6, - 0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b, - 0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c, - 0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac, - 0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2, - 0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9, - 0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1, - 0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67, - 0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe, - 0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90, - 0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e, - 0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc, - 0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d, - 0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9, - 0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6, - 0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed, - 0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42, - 0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde, - 0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6, - 0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc, - 0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6, - 0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf, - 0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7, - 0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc, - 0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00, + 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b, + 0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c, + 0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55, + 0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5, + 0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10, + 0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c, + 0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda, + 0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6, + 0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82, + 0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39, + 0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1, + 0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c, + 0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c, + 0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a, + 0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f, + 0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc, + 0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b, + 0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4, + 0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c, + 0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce, + 0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87, + 0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19, + 0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72, + 0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a, + 0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65, + 0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07, + 0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54, + 0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b, + 0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33, + 0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9, + 0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c, + 0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39, + 0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce, + 0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d, + 0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90, + 0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a, + 0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98, + 0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c, + 0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a, + 0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42, + 0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81, + 0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5, + 0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9, + 0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c, + 0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41, + 0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06, + 0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11, + 0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd, + 0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad, + 0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b, + 0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99, + 0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4, + 0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2, + 0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0, + 0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3, + 0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35, + 0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a, + 0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b, + 0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51, + 0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71, + 0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1, + 0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0, + 0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea, + 0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88, + 0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0, + 0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda, + 0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2, + 0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9, + 0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb, + 0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -2985,6 +3044,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x50 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x48 + } if len(m.ColumnKeys) > 0 { for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ColumnKeys[iNdEx]) @@ -3104,6 +3173,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x58 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x50 + } if len(m.StringValues) > 0 { for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.StringValues[iNdEx]) @@ -3446,6 +3525,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x30 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x28 + } if m.Block != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.Block)) i-- @@ -3509,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x30 + } if len(m.ColumnIDs) > 0 { dAtA36 := make([]byte, len(m.ColumnIDs)*10) var j35 int @@ -4080,6 +4174,12 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4132,6 +4232,12 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4270,6 +4376,12 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPublic(uint64(m.Block)) } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4306,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7525,6 +7640,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -7932,6 +8085,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -8771,6 +8962,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -9016,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 81220fbfe..698581215 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -116,6 +116,7 @@ message QueryResult { message ImportRequest { string Index = 1; + string Field = 2; uint64 Shard = 3; repeated uint64 RowIDs = 4; @@ -123,6 +124,8 @@ message ImportRequest { repeated string RowKeys = 7; repeated string ColumnKeys = 8; repeated int64 Timestamps = 6; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } message ImportValueRequest { @@ -134,6 +137,8 @@ message ImportValueRequest { repeated int64 Values = 6; repeated double FloatValues = 8; repeated string StringValues = 9; + int64 IndexCreatedAt = 10; + int64 FieldCreatedAt = 11; } message TranslateKeysRequest { @@ -166,6 +171,9 @@ message ImportRoaringRequest { repeated ImportRoaringRequestView views = 2; string Action = 3; uint64 Block = 4; + int64 IndexCreatedAt = 5; + int64 FieldCreatedAt = 6; + } message ImportColumnAttrsRequest { @@ -174,4 +182,5 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; + int64 IndexCreatedAt = 6; } diff --git a/pilosa.go b/pilosa.go index d7eb2f3c8..53733d9ce 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,6 +17,7 @@ package pilosa import ( "encoding/json" "regexp" + "time" "github.com/pkg/errors" ) @@ -65,6 +66,9 @@ var ( // we won't need this error at all by 2.0 though. ErrClusterDoesNotOwnShard = errors.New("node does not own shard") + // ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match + ErrPreconditionFailed = errors.New("precondition failed") + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") @@ -124,6 +128,15 @@ func newNotFoundError(err error) NotFoundError { return NotFoundError{err} } +type PreconditionFailedError struct { + error +} + +// newPreconditionFailedError returns err wrapped in a PreconditionFailedError. +func newPreconditionFailedError(err error) PreconditionFailedError { + return PreconditionFailedError{err} +} + // Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) @@ -191,6 +204,10 @@ func stringSlicesAreEqual(a, b []string) bool { return true } +func timestamp() int64 { + return time.Now().UnixNano() +} + // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/server.go b/server.go index fd07ec0b2..9fbb30811 100644 --- a/server.go +++ b/server.go @@ -356,14 +356,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { } s.executor = newExecutor(executorOpts...) - // s.holder.translateFile.logger = s.logger - path, err := expandDirName(s.dataDir) if err != nil { return nil, err } s.holder.Path = path - // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger s.holder.Stats.SetLogger(s.logger) @@ -716,10 +713,13 @@ func (s *Server) receiveMessage(m Message) error { } case *CreateIndexMessage: opt := obj.Meta - _, err := s.holder.CreateIndex(obj.Index, *opt) + idx, err := s.holder.CreateIndex(obj.Index, *opt) if err != nil { return err } + idx.mu.Lock() + idx.createdAt = obj.CreatedAt + idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err @@ -730,10 +730,13 @@ func (s *Server) receiveMessage(m Message) error { return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta - _, err := idx.createFieldIfNotExists(obj.Field, opt) + fld, err := idx.createFieldIfNotExists(obj.Field, opt) if err != nil { return err } + fld.mu.Lock() + fld.createdAt = obj.CreatedAt + fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { @@ -767,6 +770,12 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if !s.isCoordinator { + if obj.Schema != nil { + s.holder.applyCreatedAt(obj.Schema.Indexes) + } + } + case *ResizeInstruction: err := s.cluster.followResizeInstruction(obj) if err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index b01b15c59..479856fad 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -195,19 +195,81 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - body := w.Body.String() - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} -`, pilosa.ShardWidth) + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } }) + t.Run("Import", func(t *testing.T) { + indexInfo := cmd.API.Schema(context.Background()) + err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false) + if err != nil { + t.Fatalf("applying schema: %v", err) + } + + idx := indexInfo[0] + fld := indexInfo[0].Fields[0] + msg := pilosa.ImportRequest{ + Index: idx.Name, + IndexCreatedAt: idx.CreatedAt, + Field: fld.Name, + FieldCreatedAt: fld.CreatedAt, + Shard: 0, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) + httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + + w := httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + if w.Code != 200 { + t.Fatalf(w.Body.String()) + } + + msg.IndexCreatedAt = -idx.CreatedAt + msg.FieldCreatedAt = -fld.CreatedAt + data, err = ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + + w = httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + + if w.Code != 412 { + t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) + } + }) + t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + + idx, err := cmd.API.Index(context.Background(), "i0") + if err != nil { + t.Fatal(err) + } + fld, err := cmd.API.Field(context.Background(), "i0", "f1") + if err != nil { + t.Fatal(err) + } + msg := pilosa.ImportRoaringRequest{ - Clear: false, + IndexCreatedAt: idx.CreatedAt(), + FieldCreatedAt: fld.CreatedAt(), + Clear: false, Views: map[string][]byte{ "": roaringData, }, @@ -217,10 +279,14 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatal(err) } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, httpReq) + if w.Code != 200 { + t.Fatalf("Unexpected response body: %s", w.Body.String()) + } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) @@ -761,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected response body: %s", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // Verify index is gone. if hldr.Index("i") != nil { @@ -779,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i").Field("f1"); f != nil { - t.Fatal("expected nil field") + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } } }) @@ -960,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create index again @@ -970,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // create field @@ -980,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field again @@ -990,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // delete field @@ -1000,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete field again @@ -1020,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete index again @@ -1030,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } }) @@ -1042,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1052,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // set some bits