scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #401 from kuba--/etag

Browse source 
Add (in memory) CreatedAt to index and fields
This commit is contained in:
Kuba Podgórski 2020-06-03 15:09:22 +02:00 • committed by GitHub
commit a8e6846e78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 1532 additions and 523 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
api.go
View file

@ -181,11 +181,18 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
if err != nil {
return nil, errors.Wrap(err, "creating index")
}
createdAt := timestamp()
index.mu.Lock()
index.createdAt = createdAt
index.mu.Unlock()
// Send the create index message to all nodes.
err = api.server.SendSync(
&CreateIndexMessage{
Index: indexName,
Meta: &options,
Index: indexName,
CreatedAt: createdAt,
Meta: &options,
})
if err != nil {
return nil, errors.Wrap(err, "sending CreateIndex message")
@ -269,14 +276,18 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
if err != nil {
return nil, errors.Wrap(err, "creating field")
}
createdAt := timestamp()
field.mu.Lock()
field.createdAt = createdAt
field.mu.Unlock()
// Send the create field message to all nodes.
err = api.server.SendSync(
&CreateFieldMessage{
Index: indexName,
Field: fieldName,
Meta: &fo,
})
err = api.server.SendSync(&CreateFieldMessage{
Index: indexName,
Field: fieldName,
CreatedAt: createdAt,
Meta: &fo,
})
if err != nil {
api.server.logger.Printf("problem sending CreateField message: %s", err)
return nil, errors.Wrap(err, "sending CreateField message")
@ -413,14 +424,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return errors.Wrap(err, "validating api method")
}
nodes := api.cluster.shardNodes(indexName, shard)
field := api.holder.Field(indexName, fieldName)
if field == nil {
index, field, err := api.indexField(indexName, fieldName, shard)
if index == nil || field == nil {
return newNotFoundError(ErrFieldNotFound)
}
errCh := make(chan error, len(nodes))
if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return newPreconditionFailedError(err)
}
nodes := api.cluster.shardNodes(indexName, shard)
errCh := make(chan error, len(nodes))
for _, node := range nodes {
node := node
if node.ID == api.server.nodeID {
@ -803,6 +817,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
return errors.Wrap(err, "validating api method")
}
// set CreatedAt for indexes and fields (if empty), and then apply schema.
for _, index := range s.Indexes {
if index.CreatedAt == 0 {
index.CreatedAt = timestamp()
}
for _, field := range index.Fields {
if field.CreatedAt == 0 {
field.CreatedAt = timestamp()
}
}
}
if !remote {
nodes := api.cluster.Nodes()
for i, node := range nodes {
@ -813,7 +839,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
}
}
return api.holder.applySchema(s)
return errors.Wrap(api.holder.applySchema(s), "applying schema")
}
// Views returns the views in the given field.
@ -999,16 +1025,20 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
return errors.Wrap(err, "validating api method")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
// Set up import options.
options, err := setUpImportOptions(opts...)
if err != nil {
return errors.Wrap(err, "setting up import options")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
span.LogKV(
"index", req.Index,
"field", req.Field)
@ -1115,7 +1145,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
return errors.Wrap(err, "validating api method")
}
if err := req.Validate(); err != nil {
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
@ -1125,7 +1160,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
return errors.Wrap(err, "setting up import options")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
index, field, err = api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
@ -1266,6 +1301,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq
return errors.Wrap(err, "validating shard ownership")
}
if req.IndexCreatedAt != 0 {
if index.CreatedAt() != req.IndexCreatedAt {
return ErrPreconditionFailed
}
}
bulkAttrs := make(map[uint64]map[string]interface{})
for n := 0; n < len(req.ColumnIDs); n++ {
bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}

81
api_test.go
View file

@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
m1 := c[1]
t.Run("ImportColumnAttrs", func(t *testing.T) {
ctx := context.Background()
index := "i"
field := "f"
indexName := "i"
fieldName := "f"
attrKey := "k"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field)
_, err = m0.API.CreateField(ctx, indexName, fieldName)
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
columnIDs0 = append(columnIDs0, uint64(n))
val0 := attrFun(uint64(n))
attrVals0 = append(attrVals0, val0)
setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field)
if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil {
setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
t.Fatal(err)
}
columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
val1 := attrFun(uint64(n + ShardWidth))
attrVals1 = append(attrVals1, val1)
setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field)
if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil {
setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
t.Fatal(err)
}
}
// send shard0 to node1
req := &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs0,
AttrVals: attrVals0,
Shard: 0,
Index: index,
AttrKey: attrKey,
ColumnIDs: columnIDs0,
AttrVals: attrVals0,
Shard: 0,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
// send shard1 to node0
req = &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs1,
AttrVals: attrVals1,
Shard: 1,
Index: index,
AttrKey: attrKey,
ColumnIDs: columnIDs1,
AttrVals: attrVals1,
Shard: 1,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
}
// Query node0.
pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
}
}
// Query node1.
pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
@ -186,17 +188,24 @@ func TestAPI_Import(t *testing.T) {
t.Run("RowIDColumnKey", func(t *testing.T) {
ctx := context.Background()
index := "rick"
field := "f"
indexName := "rick"
fieldName := "f"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if index.CreatedAt() == 0 {
t.Fatal("index createdAt is empty")
}
field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if field.CreatedAt() == 0 {
t.Fatal("field createdAt is empty")
}
rowID := uint64(1)
timestamp := int64(0)
@ -215,21 +224,23 @@ func TestAPI_Import(t *testing.T) {
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: index,
Field: field,
Shard: 0,
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
Field: fieldName,
FieldCreatedAt: field.CreatedAt(),
Shard: 0,
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
if err := m0.API.Import(ctx, req); err != nil {
t.Fatal(err)
}
pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
@ -237,7 +248,7 @@ func TestAPI_Import(t *testing.T) {
// Query node1.
if err := test.RetryUntil(5*time.Second, func() error {
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
return err
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
return fmt.Errorf("unexpected column keys: %#v", keys)

27
cluster.go
View file

@ -610,6 +610,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus {
ClusterID: c.id,
State: c.state,
Nodes: c.nodes,
Schema: &Schema{Indexes: c.holder.Schema()},
}
}
@ -1272,9 +1273,7 @@ func (c *cluster) listenForJoins() {
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
var setNormal bool
for {
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
@ -2146,7 +2145,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
}
var availableShards *roaring.Bitmap
for _, idx := range ns.Schema.Indexes {
is := &IndexStatus{Name: idx.Name}
is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
for _, f := range idx.Fields {
if field := c.holder.Field(idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards()
@ -2155,6 +2154,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
}
is.Fields = append(is.Fields, &FieldStatus{
Name: f.Name,
CreatedAt: f.CreatedAt,
AvailableShards: availableShards,
})
}
@ -2450,6 +2450,7 @@ type ClusterStatus struct {
ClusterID string
State string
Nodes []*Node
Schema *Schema
}
// ResizeInstruction contains the instruction provided to a node
@ -2491,7 +2492,7 @@ type translationResizeNode struct {
// Schema contains information about indexes and their configuration.
type Schema struct {
Indexes []*IndexInfo
Indexes []*IndexInfo `json:"indexes"`
}
func encodeTopology(topology *Topology) *internal.Topology {
@ -2529,8 +2530,9 @@ type CreateShardMessage struct {
// CreateIndexMessage is an internal message indicating index creation.
type CreateIndexMessage struct {
Index string
Meta *IndexOptions
Index string
CreatedAt int64
Meta *IndexOptions
}
// DeleteIndexMessage is an internal message indicating index deletion.
@ -2540,9 +2542,10 @@ type DeleteIndexMessage struct {
// CreateFieldMessage is an internal message indicating field creation.
type CreateFieldMessage struct {
Index string
Field string
Meta *FieldOptions
Index string
Field string
CreatedAt int64
Meta *FieldOptions
}
// DeleteFieldMessage is an internal message indicating field deletion.
@ -2605,13 +2608,15 @@ type NodeStatus struct {
// IndexStatus is an internal message representing the contents of an index.
type IndexStatus struct {
Name string
Fields []*FieldStatus
Name string
CreatedAt int64
Fields []*FieldStatus
}
// FieldStatus is an internal message representing the contents of a field.
type FieldStatus struct {
Name string
CreatedAt int64
AvailableShards *roaring.Bitmap
}

176
docs/api-reference.md
View file

@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user
```
``` response
{
"fields": [
{
"name": "event",
"options": {
"keys": false,
"timeQuantum": "YMD",
"type": "time"
}
}
],
"name": "user",
"options": {
"keys": false,
"trackExistence": true
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
}
],
"shardWidth": 1048576
}
```
@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option
curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
```
``` response
{"success":true}
{"success":true,"name":"user","createdAt":1591179042178854000}
```
### Remove index
@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they
exist must also contain the same number of items as rows and columns. The
column IDs must all be in the shard specified in the request.
Some endpoints and data structures include a `CreatedAt` fields.
This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
but to serve as a unique identifier for use in cache invalidation.
The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
This is true except in cases where an index or field gets deleted and then recreated,
or if Pilosa is restored from a backup.
So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
and the ingester will know that it needs to drop its cache.
```
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
repeated int64 Timestamps = 6;
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated int64 Timestamps = 6;
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
int64 IndexCreatedAt = 9;
int64 FieldCreatedAt = 10;
}
```
### Create field
`POST /index/<index-name>/field/<field-name>`
@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \
-d '{"options": {"type": "int", "min": -1000, "max":2000}}'
```
``` response
{"success":true}
{"success":true,"name":"quantity","createdAt":1591180110914425000}
```
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit,
curl localhost:10101/index/user/field/language -X POST
```
``` response
{"success":true}
{"success":true,"name":"language","createdAt":1591180128294321000}
```
``` request
curl localhost:10101/index/repository/field/stats \
-X POST \
-d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
-d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
```
``` response
{"success":true}
{"success":true,"name":"stats","createdAt":1591180737881627000}
```
### Remove field
@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema
```
``` response
{
"indexes": [
"indexes": [
{
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"fields": [
{
"name": "event",
"options": {
"keys": false,
"timeQuantum": "YMD",
"type": "time"
}
},
{
"name": "language",
"options": {
"cacheSize": 50000,
"cacheType": "ranked",
"keys": false,
"type": "set"
}
}
],
"name": "user",
"options": {
"keys": false,
"trackExistence": true
}
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "language",
"createdAt": 1591180128294321000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "quantity",
"createdAt": 1591180110914425000,
"options": {
"type": "int",
"base": 0,
"bitDepth": 0,
"min": -1000,
"max": 2000,
"keys": false,
"foreignIndex": ""
}
}
]
],
"shardWidth": 1048576
}
]
}
```
@ -304,7 +340,7 @@ Returns the version of the Pilosa server.
curl -XGET localhost:10101/version
```
``` response
{"version":"v0.6.0"}
{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
```
### Get status
@ -318,19 +354,25 @@ curl -XGET localhost:10101/status
```
```response
{
"localID": "d3369125-29d8-4305-a351-b4474d14a542",
"nodes": [
{
"id": "d3369125-29d8-4305-a351-b4474d14a542",
"isCoordinator": true,
"uri": {
"host": "localhost",
"port": 10101,
"scheme": "http"
}
}
],
"state": "NORMAL"
"state": "NORMAL",
"nodes": [
{
"id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
"uri": {
"scheme": "http",
"host": "localhost",
"port": 10101
},
"grpc-uri": {
"scheme": "http",
"host": "localhost",
"port": 20101
},
"isCoordinator": true,
"state": "READY"
}
],
"localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
}
```

105
encoding/proto/proto.go
View file

@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp
func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
return &internal.ImportRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
RowIDs: m.RowIDs,
ColumnIDs: m.ColumnIDs,
RowKeys: m.RowKeys,
ColumnKeys: m.ColumnKeys,
Timestamps: m.Timestamps,
Index: m.Index,
Field: m.Field,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Shard: m.Shard,
RowIDs: m.RowIDs,
ColumnIDs: m.ColumnIDs,
RowKeys: m.RowKeys,
ColumnKeys: m.ColumnKeys,
Timestamps: m.Timestamps,
}
}
func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
return &internal.ImportValueRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
StringValues: m.StringValues,
Index: m.Index,
Field: m.Field,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
StringValues: m.StringValues,
}
}
@ -438,20 +442,23 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *
i++
}
return &internal.ImportRoaringRequest{
Clear: m.Clear,
Action: m.Action,
Block: uint64(m.Block),
Views: views,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Clear: m.Clear,
Action: m.Action,
Block: uint64(m.Block),
Views: views,
}
}
func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest {
return &internal.ImportColumnAttrsRequest{
Index: m.Index,
Shard: m.Shard,
AttrKey: m.AttrKey,
AttrVals: m.AttrVals,
ColumnIDs: m.ColumnIDs,
Index: m.Index,
IndexCreatedAt: m.IndexCreatedAt,
Shard: m.Shard,
AttrKey: m.AttrKey,
AttrVals: m.AttrVals,
ColumnIDs: m.ColumnIDs,
}
}
@ -593,9 +600,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index
func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index {
return &internal.Index{
Name: idx.Name,
Options: s.encodeIndexMeta(&idx.Options),
Fields: s.encodeFieldInfos(idx.Fields),
Name: idx.Name,
CreatedAt: idx.CreatedAt,
Options: s.encodeIndexMeta(&idx.Options),
Fields: s.encodeFieldInfos(idx.Fields),
}
}
@ -609,9 +617,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field {
func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field {
ifield := &internal.Field{
Name: f.Name,
Meta: s.encodeFieldOptions(&f.Options),
Views: make([]string, 0, len(f.Views)),
Name: f.Name,
CreatedAt: f.CreatedAt,
Meta: s.encodeFieldOptions(&f.Options),
Views: make([]string, 0, len(f.Views)),
}
for _, viewinfo := range f.Views {
@ -672,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust
State: m.State,
ClusterID: m.ClusterID,
Nodes: s.encodeNodes(m.Nodes),
Schema: s.encodeSchema(m.Schema),
}
}
@ -685,8 +695,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte
func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage {
return &internal.CreateIndexMessage{
Index: m.Index,
Meta: s.encodeIndexMeta(m.Meta),
Index: m.Index,
CreatedAt: m.CreatedAt,
Meta: s.encodeIndexMeta(m.Meta),
}
}
@ -705,9 +716,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte
func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage {
return &internal.CreateFieldMessage{
Index: m.Index,
Field: m.Field,
Meta: s.encodeFieldOptions(m.Meta),
Index: m.Index,
Field: m.Field,
CreatedAt: m.CreatedAt,
Meta: s.encodeFieldOptions(m.Meta),
}
}
@ -786,8 +798,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus
func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus {
return &internal.IndexStatus{
Name: m.Name,
Fields: s.encodeFieldStatuses(m.Fields),
Name: m.Name,
CreatedAt: m.CreatedAt,
Fields: s.encodeFieldStatuses(m.Fields),
}
}
@ -802,6 +815,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind
func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus {
return &internal.FieldStatus{
Name: m.Name,
CreatedAt: m.CreatedAt,
AvailableShards: m.AvailableShards.Slice(),
}
}
@ -938,6 +952,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo)
func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) {
m.Name = idx.Name
m.CreatedAt = idx.CreatedAt
m.Options = pilosa.IndexOptions{}
s.decodeIndexMeta(idx.Options, &m.Options)
m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields))
@ -953,6 +968,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) {
func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) {
m.Name = f.Name
m.CreatedAt = f.CreatedAt
m.Options = pilosa.FieldOptions{}
s.decodeFieldOptions(f.Meta, &m.Options)
m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views))
@ -992,6 +1008,8 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl
m.ClusterID = cs.ClusterID
m.Nodes = make([]*pilosa.Node, len(cs.Nodes))
s.decodeNodes(cs.Nodes, m.Nodes)
m.Schema = &pilosa.Schema{}
s.decodeSchema(cs.Schema, m.Schema)
}
func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) {
@ -1016,6 +1034,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m
func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) {
m.Index = pb.Index
m.CreatedAt = pb.CreatedAt
m.Meta = &pilosa.IndexOptions{}
s.decodeIndexMeta(pb.Meta, m.Meta)
}
@ -1034,6 +1053,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m
func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) {
m.Index = pb.Index
m.Field = pb.Field
m.CreatedAt = pb.CreatedAt
m.Meta = &pilosa.FieldOptions{}
s.decodeFieldOptions(pb.Meta, m.Meta)
}
@ -1107,6 +1127,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind
func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) {
m.Name = pb.Name
m.CreatedAt = pb.CreatedAt
m.Fields = s.decodeFieldStatuses(pb.Fields)
}
@ -1121,6 +1142,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie
func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) {
m.Name = pb.Name
m.CreatedAt = pb.CreatedAt
m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...)
}
@ -1149,6 +1171,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im
m.RowKeys = pb.RowKeys
m.ColumnKeys = pb.ColumnKeys
m.Timestamps = pb.Timestamps
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) {
@ -1160,6 +1184,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m
m.Values = pb.Values
m.FloatValues = pb.FloatValues
m.StringValues = pb.StringValues
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {
@ -1171,10 +1197,13 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest
m.Action = pb.Action
m.Block = int(pb.Block)
m.Views = views
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) {
m.Index = pb.Index
m.IndexCreatedAt = pb.IndexCreatedAt
m.Shard = pb.Shard
m.AttrKey = pb.AttrKey
m.AttrVals = pb.AttrVals

24
field.go
View file

@ -86,10 +86,11 @@ var availableShardFileFlushDuration = &protected{
// Field represents a container for views.
type Field struct {
mu sync.RWMutex
path string
index string
name string
mu sync.RWMutex
createdAt int64
path string
index string
name string
viewMap map[string]*view
@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) {
// Name returns the name the field was initialized with.
func (f *Field) Name() string { return f.name }
// CreatedAt is an timestamp for a specific version of field.
func (f *Field) CreatedAt() int64 {
f.mu.RLock()
defer f.mu.RUnlock()
return f.createdAt
}
// Index returns the index name the field was initialized with.
func (f *Field) Index() string { return f.index }
@ -1971,9 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FieldInfo represents schema information for a field.
type FieldInfo struct {
Name string `json:"name"`
Options FieldOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
Name string `json:"name"`
CreatedAt int64 `json:"createdAt,omitempty"`
Options FieldOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type fieldInfoSlice []*FieldInfo

10
gossip/gossip.go
View file

@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte {
Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
}
for _, idx := range m.Schema.Indexes {
is := &pilosa.IndexStatus{Name: idx.Name}
is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
for _, f := range idx.Fields {
availableShards := roaring.NewBitmap()
if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards()
}
is.Fields = append(is.Fields, &pilosa.FieldStatus{
fs := &pilosa.FieldStatus{
Name: f.Name,
CreatedAt: f.CreatedAt,
AvailableShards: availableShards,
})
}
is.Fields = append(is.Fields, fs)
}
m.Indexes = append(m.Indexes, is)
}

75
handler.go
View file

@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{}
// ImportValueRequest describes the import request structure
// for a value (BSI) import.
type ImportValueRequest struct {
Index string
Field string
Index string
IndexCreatedAt int64
Field string
FieldCreatedAt int64
// if Shard is MaxUint64 (an impossible shard value), this
// indicates that the column IDs may come from multiple shards.
Shard uint64
@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) {
// Validate ensures that the payload of the request is valid.
func (ivr *ImportValueRequest) Validate() error {
return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt)
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ivr.Index == "" || ivr.Field == "" {
return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
}
@ -160,30 +167,48 @@ func (ivr *ImportValueRequest) Validate() error {
if valueSetCount > 1 {
return errors.Errorf("must pass ints, floats, or strings but not multiple")
}
if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 {
if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
// ImportColumnAttrsRequest describes the import request structure
// for a ColumnAttr import
type ImportColumnAttrsRequest struct {
AttrKey string
ColumnIDs []uint64
AttrVals []string
Shard int64
Index string
AttrKey string
ColumnIDs []uint64
AttrVals []string
Shard int64
Index string
IndexCreatedAt int64
}
// ImportRequest describes the import request structure
// for an import.
type ImportRequest struct {
Index string
Field string
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
Index string
IndexCreatedAt int64
Field string
FieldCreatedAt int64
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
const (
@ -195,10 +220,22 @@ const (
// ImportRoaringRequest describes the import request structure
// for an import containing roaring-encoded data.
type ImportRoaringRequest struct {
Clear bool
Action string // [set, clear, overwrite]
Block int
Views map[string][]byte
IndexCreatedAt int64
FieldCreatedAt int64
Clear bool
Action string // [set, clear, overwrite]
Block int
Views map[string][]byte
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 {
if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
// ImportResponse is the structured response of an import.

81
holder.go
View file

@ -234,7 +234,13 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "opening index")
}
if err := index.Open(); err != nil {
if h.isCoordinator() {
index.createdAt = timestamp()
err = index.OpenWithTimestamp()
} else {
err = index.Open()
}
if err != nil {
if err == ErrName {
h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
continue
@ -380,11 +386,16 @@ func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{
Name: index.Name(),
Options: index.Options(),
Name: index.Name(),
CreatedAt: index.CreatedAt(),
Options: index.Options(),
}
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
fi := &FieldInfo{
Name: field.Name(),
CreatedAt: field.CreatedAt(),
Options: field.Options(),
}
for _, view := range field.views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo {
for _, index := range h.Indexes() {
di := &IndexInfo{
Name: index.Name(),
CreatedAt: index.CreatedAt(),
Options: index.Options(),
ShardWidth: ShardWidth,
}
@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo {
if strings.HasPrefix(field.name, "_") {
continue
}
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
fi := &FieldInfo{
Name: field.Name(),
CreatedAt: field.CreatedAt(),
Options: field.Options(),
}
di.Fields = append(di.Fields, fi)
}
sort.Sort(fieldInfoSlice(di.Fields))
@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo {
// applySchema applies an internal Schema to Holder.
func (h *Holder) applySchema(schema *Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
idx, err := h.CreateIndexIfNotExists(index.Name, index.Options)
for _, i := range schema.Indexes {
idx, err := h.CreateIndexIfNotExists(i.Name, i.Options)
if err != nil {
return errors.Wrap(err, "creating index")
}
if i.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = i.CreatedAt
idx.mu.Unlock()
}
// Create fields that don't exist.
for _, f := range index.Fields {
field, err := idx.createFieldIfNotExists(f.Name, &f.Options)
for _, f := range i.Fields {
fld, err := idx.createFieldIfNotExists(f.Name, &f.Options)
if err != nil {
return errors.Wrap(err, "creating field")
}
if f.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = f.CreatedAt
fld.mu.Unlock()
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := field.createViewIfNotExists(v.Name)
_, err := fld.createViewIfNotExists(v.Name)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -447,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error {
return nil
}
func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
for _, ii := range indexes {
idx := h.Index(ii.Name)
if idx == nil {
continue
}
if ii.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = ii.CreatedAt
idx.mu.Unlock()
}
for _, fi := range ii.Fields {
fld := idx.Field(fi.Name)
if fld == nil {
continue
}
if fi.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = fi.CreatedAt
fld.mu.Unlock()
}
}
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -652,6 +706,13 @@ func (h *Holder) recalculateCaches() {
}
}
func (h *Holder) isCoordinator() bool {
if s, ok := h.broadcaster.(*Server); ok {
return s.isCoordinator
}
return false
}
// setFileLimit attempts to set the open file limit to the FileLimit constant defined above.
func (h *Holder) setFileLimit() {
oldLimit := &syscall.Rlimit{}

260
http/handler.go
View file

@ -15,13 +15,13 @@
package http
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"expvar"
"fmt"
"io"
"io/ioutil"
"math"
"net"
"net/http"
@ -401,9 +401,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// successResponse is a general success/error struct for http responses.
type successResponse struct {
h *Handler
Success bool `json:"success"`
Error *Error `json:"error,omitempty"`
h *Handler
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
Error *Error `json:"error,omitempty"`
}
// check determines success or failure based on the error.
@ -504,7 +506,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
schema := h.api.Schema(r.Context())
if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here?
if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
h.logger.Printf("write schema response error: %s", err)
}
}
@ -773,7 +775,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
return
}
resp := successResponse{h: h}
resp := successResponse{h: h, Name: indexName}
// Decode request.
req := postIndexRequest{
@ -787,8 +789,15 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
return
}
_, err = h.api.CreateIndex(r.Context(), indexName, req.Options)
index, err := h.api.CreateIndex(r.Context(), indexName, req.Options)
if index != nil {
resp.CreatedAt = index.CreatedAt()
} else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok {
if index, _ = h.api.Index(r.Context(), indexName); index != nil {
resp.CreatedAt = index.CreatedAt()
}
}
resp.write(w, err)
}
@ -853,7 +862,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
return
}
resp := successResponse{h: h}
resp := successResponse{h: h, Name: fieldName}
// Decode request.
var req postFieldRequest
@ -925,11 +934,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex))
}
_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...)
if _, ok := err.(pilosa.BadRequestError); ok {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if field != nil {
resp.CreatedAt = field.CreatedAt()
} else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok {
if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil {
resp.CreatedAt = field.CreatedAt()
}
}
resp.write(w, err)
}
@ -1240,7 +1256,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error
// readProtobufQueryRequest parses query parameters in protobuf from r.
func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) {
// Slurp the body.
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1258,7 +1274,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
q := r.URL.Query()
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
buf, err := readBody(r)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1319,95 +1335,14 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse
return json.NewEncoder(w).Encode(resp)
}
// handlePostImport handles /import requests.
func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
func validateProtobufHeader(r *http.Request) (error string, code int) {
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
return
return "Unsupported media type", http.StatusUnsupportedMediaType
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
// If the clear flag is true, treat the import as clear bits.
q := r.URL.Query()
doClear := q.Get("clear") == "true"
doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true"
opts := []pilosa.ImportOption{
pilosa.OptImportOptionsClear(doClear),
pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck),
}
// Get index and field type to determine how to handle the
// import data.
field, err := h.api.Field(r.Context(), indexName, fieldName)
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrIndexNotFound:
fallthrough
case pilosa.ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Read entire body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Unmarshal request based on field type.
if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
// Field type: Int
// Marshal into request object.
req := &pilosa.ImportValueRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.ImportValue(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
} else {
// Field type: set, time, mutex
// Marshal into request object.
req := &pilosa.ImportRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.Import(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
}
// Write response.
_, err = w.Write(importOk)
if err != nil {
h.logger.Printf("writing import response: %v", err)
if r.Header.Get("Accept") != "application/x-protobuf" {
return "Not acceptable", http.StatusNotAcceptable
}
return
}
// handleGetExport handles /export requests.
@ -1885,6 +1820,95 @@ func GetHTTPClient(t *tls.Config) *http.Client {
return &http.Client{Transport: transport}
}
// handlePostImport handles /import requests.
func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
// Get index and field type to determine how to handle the
// import data.
indexName := mux.Vars(r)["index"]
index, err := h.api.Index(r.Context(), indexName)
if err != nil {
if errors.Cause(err) == pilosa.ErrIndexNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
fieldName := mux.Vars(r)["field"]
field := index.Field(fieldName)
if field == nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// If the clear flag is true, treat the import as clear bits.
q := r.URL.Query()
doClear := q.Get("clear") == "true"
doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true"
opts := []pilosa.ImportOption{
pilosa.OptImportOptionsClear(doClear),
pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck),
}
// Read entire body.
body, err := readBody(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Unmarshal request based on field type.
if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
// Field type: Int
// Marshal into request object.
req := &pilosa.ImportValueRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.ImportValue(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
} else {
// Field type: set, time, mutex
// Marshal into request object.
req := &pilosa.ImportRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.Import(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
}
// Write response.
_, err = w.Write(importOk)
if err != nil {
h.logger.Printf("writing import response: %v", err)
}
}
// handlePostImportColumnAttrs
func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
@ -1898,7 +1922,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
opts := []pilosa.ImportOption{}
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -1911,7 +1935,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
}
if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
@ -1922,16 +1951,16 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
}
}
// handlPostRoaringImport
// handlePostImportRoaring
func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
// Get index and field type to determine how to handle the
// import data.
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
@ -1946,7 +1975,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
if err != nil {
@ -1976,6 +2005,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
resp.Err = err.Error()
if _, ok := err.(pilosa.BadRequestError); ok {
w.WriteHeader(http.StatusBadRequest)
} else if _, ok := err.(pilosa.NotFoundError); ok {
w.WriteHeader(http.StatusNotFound)
} else if _, ok := err.(pilosa.PreconditionFailedError); ok {
w.WriteHeader(http.StatusPreconditionFailed)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
@ -2040,3 +2073,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request)
h.logger.Printf("writing translate keys response: %v", err)
}
}
// Read entire request body.
func readBody(r *http.Request) ([]byte, error) {
var contentLength int64 = bytes.MinRead
if r.ContentLength > 0 {
contentLength = r.ContentLength
}
buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength))
if _, err := buf.ReadFrom(r.Body); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

31
index.go
View file

@ -36,10 +36,11 @@ import (
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
path string
name string
keys bool // use string keys
mu sync.RWMutex
createdAt int64
path string
name string
keys bool // use string keys
// Existence tracking.
trackExistence bool
@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) {
}, nil
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()
defer i.mu.RUnlock()
return i.createdAt
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions {
}
// Open opens and initializes the index.
func (i *Index) Open() (err error) {
func (i *Index) Open() error { return i.open(false) }
// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (i *Index) OpenWithTimestamp() error { return i.open(true) }
func (i *Index) open(withTimestamp bool) (err error) {
// Ensure the path exists.
i.logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
@ -154,7 +167,7 @@ func (i *Index) Open() (err error) {
}
i.logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(); err != nil {
if err := i.openFields(withTimestamp); err != nil {
return errors.Wrap(err, "opening fields")
}
@ -197,7 +210,7 @@ func (i *Index) Open() (err error) {
var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields() error {
func (i *Index) openFields(withTimestamp bool) error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -229,6 +242,9 @@ fileLoop:
i.logger.Debugf("open field: %s", fi.Name())
mu.Lock()
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
fld.createdAt = timestamp()
}
mu.Unlock()
if err != nil {
return errors.Wrapf(ErrName, "'%s'", fi.Name())
@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
CreatedAt int64 `json:"createdAt,omitempty"`
Options IndexOptions `json:"options"`
Fields []*FieldInfo `json:"fields"`
ShardWidth uint64 `json:"shardWidth"`

453
internal/private.pb.go
View file

@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string {
type CreateIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta {
return nil
}
func (m *CreateIndexMessage) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type CreateFieldMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"`
CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions {
return nil
}
func (m *CreateFieldMessage) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type DeleteFieldMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -853,6 +869,7 @@ type Field struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"`
Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"`
CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -912,6 +929,13 @@ func (m *Field) GetViews() []string {
return nil
}
func (m *Field) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type Schema struct {
Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index {
type Index struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"`
Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -1008,6 +1033,13 @@ func (m *Index) GetName() string {
return ""
}
func (m *Index) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
func (m *Index) GetOptions() *IndexMeta {
if m != nil {
return m.Options
@ -1340,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus {
type IndexStatus struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1392,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus {
return nil
}
func (m *IndexStatus) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type FieldStatus struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1447,10 +1488,18 @@ func (m *FieldStatus) GetAvailableShards() []uint64 {
return nil
}
func (m *FieldStatus) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type ClusterStatus struct {
ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"`
Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1510,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node {
return nil
}
func (m *ClusterStatus) GetSchema() *Schema {
if m != nil {
return m.Schema
}
return nil
}
type BSIGroup struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"`
@ -2421,96 +2477,99 @@ func init() {
func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
var fileDescriptor_d2a91b51c7bdc125 = []byte{
// 1418 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0x1b, 0xc5,
0x13, 0xff, 0xaf, 0x56, 0xb6, 0xa4, 0x96, 0xe5, 0xc8, 0x93, 0xc4, 0xd9, 0xf8, 0x4f, 0x19, 0x31,
0xa4, 0x88, 0x48, 0x15, 0x26, 0x95, 0x50, 0xc5, 0x67, 0xaa, 0x12, 0x5b, 0x4e, 0x10, 0xc1, 0x8e,
0x33, 0x72, 0x72, 0xe3, 0x30, 0x5e, 0x4d, 0xc5, 0x5b, 0x5e, 0xed, 0x8a, 0xdd, 0x59, 0x47, 0xce,
0x81, 0x2b, 0x54, 0xf1, 0x02, 0x1c, 0x38, 0xf0, 0x1e, 0xbc, 0x00, 0x47, 0x1e, 0x81, 0x0a, 0x4f,
0xc1, 0x8d, 0x9a, 0x9e, 0x99, 0xdd, 0x95, 0x2c, 0xe3, 0x90, 0x70, 0xdb, 0xfe, 0xf5, 0x77, 0x4f,
0x77, 0xcf, 0x2c, 0xb4, 0xc6, 0x49, 0x70, 0xcc, 0xa5, 0xd8, 0x18, 0x27, 0xb1, 0x8c, 0x49, 0x3d,
0x88, 0xa4, 0x48, 0x22, 0x1e, 0xae, 0x2d, 0x8d, 0xb3, 0x83, 0x30, 0xf0, 0x35, 0x4e, 0x1f, 0x40,
0xa3, 0x1f, 0x0d, 0xc5, 0x64, 0x47, 0x48, 0x4e, 0x08, 0x54, 0x1f, 0x8a, 0x93, 0xd4, 0x73, 0x3b,
0x4e, 0xb7, 0xce, 0xf0, 0x9b, 0xbc, 0x07, 0xcb, 0xfb, 0x09, 0xf7, 0x8f, 0xb6, 0x27, 0x41, 0x2a,
0x45, 0xe4, 0x0b, 0xaf, 0x8a, 0xdc, 0x19, 0x94, 0xfe, 0xe2, 0xc2, 0xd2, 0xfd, 0x40, 0x84, 0xc3,
0x47, 0x63, 0x19, 0xc4, 0x51, 0xaa, 0x8c, 0xed, 0x9f, 0x8c, 0x85, 0x57, 0xef, 0x38, 0xdd, 0x06,
0xc3, 0x6f, 0xf2, 0x16, 0x34, 0xb6, 0xb8, 0x7f, 0x28, 0x90, 0xe1, 0x22, 0xa3, 0x00, 0x72, 0xee,
0x20, 0x78, 0xa1, 0xbd, 0xb4, 0x58, 0x01, 0x90, 0x0e, 0x34, 0xf7, 0x83, 0x91, 0x78, 0x9c, 0xf1,
0x48, 0x66, 0x23, 0x6f, 0x01, 0xb5, 0xcb, 0x10, 0x59, 0x85, 0xc5, 0x47, 0xe1, 0x70, 0x27, 0x88,
0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, 0x50, 0x16, 0xe7, 0x13, 0x0f, 0x0a, 0x9c, 0x4f, 0xf2, 0x74,
0x9b, 0xd3, 0xe9, 0xee, 0xc6, 0x03, 0xc9, 0xa3, 0x21, 0x4f, 0x86, 0x4f, 0x03, 0xf1, 0xdc, 0x5b,
0xd2, 0xe9, 0x4e, 0xa3, 0x4a, 0x77, 0x93, 0xa7, 0xc2, 0x6b, 0xa1, 0x45, 0xfc, 0x26, 0x6b, 0x50,
0xdf, 0x0c, 0x64, 0x4f, 0x8c, 0xe5, 0xa1, 0xb7, 0xdc, 0x71, 0xba, 0x55, 0x96, 0xd3, 0xe4, 0x12,
0x2c, 0x0c, 0x7c, 0x1e, 0x0a, 0xef, 0x02, 0x2a, 0x68, 0x82, 0x50, 0x58, 0xba, 0x1f, 0x27, 0x22,
0x78, 0x16, 0xe1, 0x21, 0x78, 0x6d, 0x4c, 0x6a, 0x0a, 0x23, 0xef, 0x82, 0xab, 0x52, 0x5a, 0xe9,
0x38, 0xdd, 0xe6, 0xad, 0x95, 0x0d, 0x7b, 0x8e, 0x1b, 0x3d, 0xe1, 0x07, 0x23, 0x1e, 0x32, 0xc5,
0x45, 0x21, 0x3e, 0xf1, 0xc8, 0xd9, 0x42, 0x7c, 0x42, 0x29, 0x2c, 0xf7, 0x47, 0xe3, 0x38, 0x91,
0x4c, 0xa4, 0xe3, 0x38, 0x4a, 0x05, 0x69, 0x83, 0xbb, 0x9d, 0x24, 0x9e, 0x83, 0x6e, 0xd5, 0x27,
0xfd, 0x0e, 0xda, 0x9b, 0x61, 0xec, 0x1f, 0xf5, 0xb8, 0xe4, 0x4c, 0x7c, 0x9b, 0x89, 0x54, 0xaa,
0xd8, 0x75, 0x78, 0x5a, 0x4e, 0x13, 0x0a, 0xc5, 0xf3, 0xf6, 0x2a, 0x1a, 0x45, 0x42, 0xd5, 0x05,
0xab, 0xa6, 0x8f, 0x07, 0xbf, 0x31, 0xf7, 0x43, 0x9e, 0x0c, 0xf1, 0x4c, 0xab, 0x4c, 0x13, 0x0a,
0x45, 0x4f, 0xd8, 0x07, 0x55, 0xa6, 0x09, 0xda, 0x87, 0x95, 0x92, 0x7f, 0x13, 0xe6, 0x2a, 0x2c,
0xb2, 0xf8, 0x79, 0xbf, 0x97, 0x7a, 0x4e, 0xc7, 0xed, 0x56, 0x99, 0xa1, 0xb0, 0x61, 0xe2, 0x30,
0x1b, 0x45, 0x8a, 0x55, 0x41, 0x56, 0x01, 0xd0, 0xab, 0xb0, 0x80, 0xdd, 0xa3, 0xb2, 0x2c, 0x74,
0xd5, 0x27, 0xfd, 0xde, 0x81, 0xc6, 0x0e, 0x9f, 0x60, 0x20, 0x29, 0xb9, 0x03, 0x75, 0x7b, 0xb6,
0x28, 0xd4, 0xbc, 0xf5, 0x4e, 0x51, 0xc1, 0x5c, 0x6c, 0xc3, 0xca, 0x6c, 0x47, 0x32, 0x39, 0x61,
0xb9, 0xca, 0xda, 0xe7, 0xd0, 0x9a, 0x62, 0x29, 0x7f, 0x47, 0xe2, 0xc4, 0x56, 0xf5, 0x48, 0x9c,
0xa8, 0x5c, 0x8f, 0x79, 0x98, 0x09, 0xac, 0x55, 0x95, 0x69, 0xe2, 0xb3, 0xca, 0x27, 0x0e, 0x7d,
0x0a, 0x64, 0x2b, 0x11, 0x5c, 0x0a, 0x74, 0xb2, 0x23, 0xd2, 0x94, 0x3f, 0x13, 0xe7, 0x55, 0xdc,
0x2d, 0x57, 0x3c, 0xaf, 0x6e, 0xa5, 0x54, 0x5d, 0x7a, 0x03, 0x48, 0x4f, 0x84, 0x42, 0x0a, 0x33,
0xdd, 0xff, 0x60, 0x97, 0x0e, 0x6c, 0x0c, 0xe7, 0xcb, 0x92, 0xeb, 0x50, 0x55, 0xab, 0x02, 0x9d,
0x35, 0x6f, 0x5d, 0x2c, 0xea, 0x94, 0x6f, 0x11, 0x86, 0x02, 0x34, 0xb4, 0x46, 0x31, 0xca, 0x57,
0x4c, 0x6c, 0xaa, 0x95, 0x6e, 0x18, 0x57, 0x2e, 0xba, 0x5a, 0x2d, 0x5c, 0x95, 0xd7, 0x8c, 0xf1,
0x76, 0xd7, 0xa6, 0xfb, 0xba, 0xde, 0xa8, 0x0f, 0xff, 0xd7, 0x16, 0xee, 0x1d, 0xf3, 0x20, 0xe4,
0x07, 0xe1, 0xbf, 0x3a, 0x91, 0xa9, 0xc0, 0x3d, 0xa8, 0xa1, 0x6e, 0xbf, 0x67, 0x7a, 0xdb, 0x92,
0xf4, 0x1b, 0x28, 0xc6, 0x64, 0x97, 0x8f, 0x84, 0xb1, 0x86, 0xdf, 0x79, 0xbe, 0x95, 0xf3, 0xf3,
0x55, 0x8e, 0xd5, 0x68, 0xa9, 0x55, 0xed, 0x2a, 0xc7, 0x48, 0xd0, 0xdb, 0xb0, 0x38, 0xf0, 0x0f,
0xc5, 0x88, 0x93, 0xf7, 0xa1, 0x86, 0x11, 0x8a, 0xd4, 0x74, 0xf4, 0x85, 0x99, 0x93, 0x62, 0x96,
0x4f, 0x53, 0x93, 0xd9, 0xdc, 0x98, 0x3e, 0x80, 0x9a, 0x71, 0x8c, 0x13, 0x7d, 0xc6, 0x89, 0x5b,
0x19, 0x72, 0x1d, 0x16, 0x31, 0xd8, 0xd4, 0xab, 0xce, 0x7a, 0x45, 0x9c, 0x19, 0x36, 0xdd, 0x06,
0xf7, 0x09, 0xeb, 0xab, 0xc1, 0xc6, 0x80, 0xad, 0x53, 0x43, 0xa9, 0x50, 0xbe, 0x8c, 0x53, 0x69,
0xca, 0x8a, 0xdf, 0x0a, 0xdb, 0x8b, 0x13, 0x89, 0x25, 0x6d, 0x31, 0xfc, 0xa6, 0x3f, 0x3b, 0x50,
0xdd, 0x8d, 0x87, 0x82, 0x2c, 0x43, 0xa5, 0xdf, 0x33, 0x46, 0x2a, 0xfd, 0x1e, 0x79, 0x1b, 0xed,
0x9b, 0x52, 0xb6, 0x8a, 0x28, 0x9e, 0xb0, 0x3e, 0x43, 0xcf, 0xd7, 0xa0, 0xd5, 0x4f, 0xb7, 0xe2,
0x38, 0x19, 0x06, 0x11, 0x97, 0x71, 0x62, 0xee, 0xbc, 0x69, 0x10, 0x67, 0x4b, 0x72, 0xa9, 0x6f,
0xa3, 0x06, 0xd3, 0x04, 0xb9, 0x0e, 0xb5, 0x07, 0x6c, 0x6f, 0x4b, 0x39, 0x58, 0x98, 0xe7, 0xc0,
0x72, 0xe9, 0x5d, 0x68, 0xab, 0xe8, 0x50, 0xcb, 0x36, 0xd2, 0x2a, 0x2c, 0x2a, 0x2c, 0x8f, 0xd6,
0x50, 0x85, 0xab, 0x4a, 0xc9, 0x15, 0xfd, 0x5a, 0x5b, 0xd8, 0x3e, 0x16, 0x91, 0x2c, 0xb5, 0x22,
0xd2, 0x68, 0xa0, 0xc5, 0x34, 0x41, 0xa8, 0xae, 0x84, 0x49, 0x79, 0xb9, 0x88, 0x48, 0xa1, 0x0c,
0x79, 0xf4, 0x47, 0x07, 0xc0, 0x06, 0x94, 0xa5, 0xb9, 0x8a, 0x73, 0xb6, 0x0a, 0xe9, 0xda, 0x96,
0x32, 0x63, 0xd8, 0x2e, 0xa4, 0x34, 0xce, 0x6c, 0xcb, 0x7d, 0x58, 0xb4, 0x9c, 0x3e, 0xfc, 0xcb,
0x33, 0xad, 0xa2, 0xbd, 0x16, 0x8d, 0xb7, 0x07, 0xcd, 0x12, 0x7e, 0x46, 0xfb, 0xd9, 0x7e, 0xaa,
0xcc, 0x9a, 0x44, 0xdc, 0x98, 0xb4, 0x5d, 0xf5, 0x10, 0x9a, 0x25, 0x78, 0xae, 0xc5, 0x2e, 0x5c,
0x98, 0x1e, 0x70, 0x7b, 0x71, 0xcc, 0xc2, 0x34, 0x80, 0xd6, 0x56, 0x98, 0xa5, 0x52, 0x24, 0xc6,
0x9c, 0xba, 0x6d, 0x34, 0x90, 0x1f, 0x5e, 0x01, 0xcc, 0x3f, 0x3f, 0x72, 0x0d, 0x16, 0x54, 0x19,
0xf5, 0x9c, 0x9e, 0xae, 0xb1, 0x66, 0xd2, 0xa7, 0x50, 0xdf, 0x1c, 0xf4, 0x1f, 0x24, 0x71, 0x36,
0x9e, 0x1b, 0xb4, 0x7d, 0x4a, 0x55, 0x4a, 0x4f, 0xa9, 0xb6, 0x7e, 0x16, 0xb8, 0xf8, 0x9c, 0xc0,
0x37, 0x40, 0x5b, 0xbf, 0x01, 0xaa, 0x06, 0xe1, 0x6a, 0xb1, 0xaf, 0xe8, 0x1d, 0xac, 0xd6, 0xc3,
0xeb, 0x6c, 0x32, 0x7b, 0x9b, 0xbb, 0xc5, 0x6d, 0xae, 0x8c, 0xea, 0x45, 0xf9, 0x5f, 0x1a, 0xfd,
0xab, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x0b, 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x27, 0x4a,
0xff, 0xab, 0xf8, 0xc0, 0x54, 0xdb, 0x65, 0x9a, 0x78, 0x95, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b,
0xdc, 0xa7, 0x45, 0xcb, 0x22, 0xe4, 0x26, 0xd4, 0x06, 0x71, 0x96, 0xf8, 0x79, 0xfb, 0x96, 0x16,
0xb0, 0x8e, 0x4c, 0xb3, 0x99, 0x15, 0x23, 0x8f, 0x81, 0xec, 0x27, 0x3c, 0x4a, 0x43, 0xae, 0x82,
0xb5, 0xca, 0xf5, 0xd9, 0x07, 0x44, 0x49, 0x66, 0xca, 0xce, 0x1c, 0x65, 0xf2, 0x51, 0x79, 0x3e,
0xbd, 0x1a, 0x46, 0x7d, 0x69, 0x3a, 0x6a, 0xd3, 0xf2, 0xe5, 0x39, 0xbe, 0x33, 0xd3, 0xa9, 0xde,
0x22, 0x2a, 0x5e, 0x29, 0x14, 0xa7, 0xd8, 0x6c, 0x5a, 0x9a, 0xfe, 0xe0, 0xc0, 0x52, 0x39, 0xb2,
0x57, 0xda, 0x0b, 0xf9, 0x81, 0x57, 0xce, 0x7f, 0xa1, 0xd8, 0x03, 0xaf, 0xce, 0x7b, 0x13, 0x2e,
0x94, 0x5f, 0x2d, 0x19, 0x5c, 0x39, 0xa3, 0x5c, 0x6f, 0x10, 0x54, 0x07, 0x9a, 0x7b, 0x3c, 0x91,
0x81, 0x32, 0x69, 0xae, 0xe4, 0x05, 0x56, 0x86, 0xe8, 0x11, 0x5c, 0x3d, 0xd5, 0x7c, 0x5b, 0xf1,
0x68, 0xac, 0xba, 0xfc, 0x0d, 0x9a, 0x50, 0x2d, 0xea, 0x24, 0x31, 0xed, 0xd7, 0x60, 0x9a, 0xa0,
0x9f, 0xc2, 0xe5, 0x81, 0x90, 0xa5, 0xd6, 0xb3, 0x33, 0xd4, 0x01, 0x77, 0x57, 0x3c, 0x3f, 0x23,
0x41, 0xc5, 0xa2, 0x5f, 0x80, 0xf7, 0x64, 0x3c, 0xe4, 0x52, 0xbc, 0x96, 0xf6, 0x26, 0xd4, 0xf7,
0xe3, 0x71, 0x1c, 0xc6, 0xcf, 0x4e, 0xce, 0xd9, 0x65, 0x1e, 0xd4, 0xf4, 0xad, 0xa4, 0x97, 0x63,
0x83, 0x59, 0x92, 0x5e, 0x54, 0x63, 0xea, 0xf3, 0xd0, 0xcf, 0x42, 0x15, 0x86, 0x7a, 0x5e, 0xa7,
0x54, 0x98, 0x41, 0xe0, 0x58, 0xb8, 0xd2, 0x45, 0x77, 0x0f, 0x01, 0x7b, 0xd1, 0x69, 0x8a, 0x7c,
0x0c, 0xcd, 0x92, 0xb4, 0x29, 0xe0, 0xe5, 0x99, 0x79, 0xd1, 0x4c, 0x56, 0x96, 0xa4, 0xbf, 0x3a,
0x53, 0x9a, 0xa7, 0xee, 0x7c, 0xe3, 0xf0, 0x58, 0x1f, 0x4a, 0x9d, 0x19, 0x4a, 0xe5, 0xba, 0x3d,
0xf1, 0xc3, 0x2c, 0x55, 0x2c, 0x7d, 0xcd, 0x17, 0x80, 0xca, 0x55, 0xfd, 0x43, 0xc6, 0x99, 0x34,
0x9b, 0xd3, 0x92, 0xea, 0x77, 0xae, 0x27, 0xf8, 0x30, 0x0c, 0x22, 0x81, 0x5d, 0xea, 0xb2, 0x9c,
0x26, 0x37, 0xf5, 0xb6, 0xb7, 0xa3, 0xb6, 0x36, 0x37, 0x7c, 0x94, 0xd0, 0x37, 0x41, 0x4a, 0x09,
0xb4, 0x67, 0x59, 0x9b, 0xed, 0xdf, 0x5e, 0xae, 0x3b, 0xbf, 0xbf, 0x5c, 0x77, 0xfe, 0x78, 0xb9,
0xee, 0xfc, 0xf4, 0xe7, 0xfa, 0xff, 0x0e, 0x16, 0xf1, 0xaf, 0xfc, 0xf6, 0xdf, 0x01, 0x00, 0x00,
0xff, 0xff, 0x8a, 0x80, 0x2a, 0x36, 0xbe, 0x0f, 0x00, 0x00,
// 1458 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45,
0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf,
0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0,
0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa,
0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0,
0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97,
0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75,
0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e,
0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec,
0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17,
0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8,
0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba,
0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91,
0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47,
0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee,
0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27,
0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78,
0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93,
0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3,
0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17,
0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29,
0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb,
0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38,
0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0,
0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45,
0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21,
0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29,
0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c,
0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c,
0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96,
0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37,
0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2,
0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87,
0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4,
0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf,
0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89,
0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a,
0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3,
0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b,
0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e,
0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf,
0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3,
0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce,
0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e,
0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce,
0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15,
0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0,
0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae,
0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79,
0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64,
0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47,
0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78,
0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82,
0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68,
0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5,
0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5,
0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b,
0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce,
0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2,
0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44,
0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae,
0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27,
0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d,
0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02,
0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56,
0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0,
0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91,
0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07,
0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38,
0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c,
0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04,
0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92,
0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2,
0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59,
0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98,
0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01,
0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e,
0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10,
0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19,
0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2,
0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d,
0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe,
0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3,
0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a,
0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9,
0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c,
0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f,
0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b,
0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb,
0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05,
0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10,
0x00, 0x00,
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@ -3043,6 +3102,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if m.Meta != nil {
{
size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i])
@ -3089,6 +3153,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x20
}
if m.Meta != nil {
{
size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i])
@ -3229,6 +3298,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x20
}
if len(m.Views) > 0 {
for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.Views[iNdEx])
@ -3351,6 +3425,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) {
dAtA[i] = 0x22
}
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x10
}
if len(m.Name) > 0 {
i -= len(m.Name)
copy(dAtA[i:], m.Name)
@ -3656,6 +3735,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if len(m.Fields) > 0 {
for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- {
{
@ -3704,6 +3788,11 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if len(m.AvailableShards) > 0 {
dAtA19 := make([]byte, len(m.AvailableShards)*10)
var j18 int
@ -3756,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.Schema != nil {
{
size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintPrivate(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x22
}
if len(m.Nodes) > 0 {
for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- {
{
@ -4759,6 +4860,9 @@ func (m *CreateIndexMessage) Size() (n int) {
l = m.Meta.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4783,6 +4887,9 @@ func (m *CreateFieldMessage) Size() (n int) {
l = m.Meta.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4852,6 +4959,9 @@ func (m *Field) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4886,6 +4996,9 @@ func (m *Index) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if len(m.Fields) > 0 {
for _, e := range m.Fields {
l = e.Size()
@ -5037,6 +5150,9 @@ func (m *IndexStatus) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5060,6 +5176,9 @@ func (m *FieldStatus) Size() (n int) {
}
n += 1 + sovPrivate(uint64(l)) + l
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5086,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.Schema != nil {
l = m.Schema.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -7021,6 +7144,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7175,6 +7317,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7584,6 +7745,25 @@ func (m *Field) Unmarshal(dAtA []byte) error {
}
m.Views = append(m.Views, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7758,6 +7938,25 @@ func (m *Index) Unmarshal(dAtA []byte) error {
}
m.Name = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType)
@ -8682,6 +8881,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8844,6 +9062,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error {
} else {
return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType)
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8996,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthPrivate
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Schema == nil {
m.Schema = &Schema{}
}
if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])

7
internal/private.proto
View file

@ -64,12 +64,14 @@ message DeleteIndexMessage {
message CreateIndexMessage {
string Index = 1;
IndexMeta Meta = 2;
int64 CreatedAt = 3;
}
message CreateFieldMessage {
string Index = 1;
string Field = 2;
FieldOptions Meta = 3;
int64 CreatedAt = 4;
}
message DeleteFieldMessage {
@ -87,6 +89,7 @@ message Field {
string Name = 1;
FieldOptions Meta = 2;
repeated string Views = 3;
int64 CreatedAt = 4;
}
message Schema {
@ -95,6 +98,7 @@ message Schema {
message Index {
string Name = 1;
int64 CreatedAt = 2;
IndexMeta Options = 5;
repeated Field Fields = 4;
}
@ -132,17 +136,20 @@ message NodeStatus {
message IndexStatus {
string Name = 1;
repeated FieldStatus Fields = 2;
int64 CreatedAt = 3;
}
message FieldStatus {
string Name = 1;
repeated uint64 AvailableShards = 2;
int64 CreatedAt = 3;
}
message ClusterStatus {
string ClusterID = 1;
string State = 2;
repeated Node Nodes = 3;
Schema Schema = 4;
}
message BSIGroup {

400
internal/public.pb.go
View file

@ -1183,6 +1183,8 @@ type ImportRequest struct {
RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"`
ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"`
Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 {
return nil
}
func (m *ImportRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type ImportValueRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -1286,6 +1302,8 @@ type ImportValueRequest struct {
Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"`
FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"`
StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string {
return nil
}
func (m *ImportValueRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportValueRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type TranslateKeysRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct {
Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"`
Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"`
Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1726,12 +1760,27 @@ func (m *ImportRoaringRequest) GetBlock() uint64 {
return 0
}
func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type ImportColumnAttrsRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"`
AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"`
AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1805,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 {
return nil
}
func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func init() {
proto.RegisterType((*Row)(nil), "internal.Row")
proto.RegisterType((*SignedRow)(nil), "internal.SignedRow")
@ -1837,83 +1893,86 @@ func init() {
func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) }
var fileDescriptor_413a91106d7bcce8 = []byte{
// 1207 bytes of a gzipped FileDescriptorProto
// 1258 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45,
0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b,
0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b,
0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00,
0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d,
0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa,
0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a,
0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b,
0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e,
0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf,
0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92,
0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a,
0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85,
0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23,
0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42,
0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3,
0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5,
0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87,
0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54,
0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b,
0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87,
0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a,
0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e,
0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36,
0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75,
0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72,
0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae,
0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c,
0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde,
0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68,
0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2,
0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c,
0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47,
0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0,
0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3,
0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e,
0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17,
0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97,
0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61,
0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb,
0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29,
0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef,
0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c,
0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30,
0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea,
0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0,
0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5,
0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c,
0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d,
0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b,
0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43,
0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6,
0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b,
0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c,
0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac,
0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2,
0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9,
0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1,
0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67,
0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe,
0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90,
0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e,
0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc,
0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d,
0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9,
0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6,
0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed,
0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42,
0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde,
0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6,
0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc,
0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6,
0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf,
0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7,
0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc,
0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00,
0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01,
0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b,
0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c,
0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55,
0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5,
0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10,
0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c,
0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda,
0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6,
0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82,
0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39,
0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1,
0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c,
0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c,
0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a,
0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f,
0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc,
0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21,
0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b,
0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4,
0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c,
0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06,
0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35,
0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40,
0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce,
0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87,
0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19,
0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72,
0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a,
0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65,
0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07,
0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54,
0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b,
0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33,
0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9,
0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c,
0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39,
0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce,
0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d,
0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90,
0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7,
0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a,
0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98,
0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5,
0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c,
0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a,
0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42,
0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81,
0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5,
0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9,
0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c,
0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41,
0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06,
0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11,
0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd,
0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad,
0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b,
0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99,
0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4,
0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2,
0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0,
0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3,
0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35,
0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a,
0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b,
0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51,
0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71,
0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1,
0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0,
0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea,
0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88,
0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0,
0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda,
0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2,
0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9,
0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb,
0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00,
0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00,
}
func (m *Row) Marshal() (dAtA []byte, err error) {
@ -2985,6 +3044,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x50
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x48
}
if len(m.ColumnKeys) > 0 {
for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.ColumnKeys[iNdEx])
@ -3104,6 +3173,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x58
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x50
}
if len(m.StringValues) > 0 {
for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.StringValues[iNdEx])
@ -3446,6 +3525,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x30
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x28
}
if m.Block != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.Block))
i--
@ -3509,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x30
}
if len(m.ColumnIDs) > 0 {
dAtA36 := make([]byte, len(m.ColumnIDs)*10)
var j35 int
@ -4080,6 +4174,12 @@ func (m *ImportRequest) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4132,6 +4232,12 @@ func (m *ImportValueRequest) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4270,6 +4376,12 @@ func (m *ImportRoaringRequest) Size() (n int) {
if m.Block != 0 {
n += 1 + sovPublic(uint64(m.Block))
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4306,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) {
}
n += 1 + sovPublic(uint64(l)) + l
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -7525,6 +7640,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
}
m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 9:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -7932,6 +8085,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 11:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -8771,6 +8962,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -9016,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error {
} else {
return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType)
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])

9
internal/public.proto
View file

@ -116,6 +116,7 @@ message QueryResult {
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
@ -123,6 +124,8 @@ message ImportRequest {
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
repeated int64 Timestamps = 6;
int64 IndexCreatedAt = 9;
int64 FieldCreatedAt = 10;
}
message ImportValueRequest {
@ -134,6 +137,8 @@ message ImportValueRequest {
repeated int64 Values = 6;
repeated double FloatValues = 8;
repeated string StringValues = 9;
int64 IndexCreatedAt = 10;
int64 FieldCreatedAt = 11;
}
message TranslateKeysRequest {
@ -166,6 +171,9 @@ message ImportRoaringRequest {
repeated ImportRoaringRequestView views = 2;
string Action = 3;
uint64 Block = 4;
int64 IndexCreatedAt = 5;
int64 FieldCreatedAt = 6;
}
message ImportColumnAttrsRequest {
@ -174,4 +182,5 @@ message ImportColumnAttrsRequest {
string AttrKey = 3;
repeated string AttrVals = 4;
repeated uint64 ColumnIDs = 5;
int64 IndexCreatedAt = 6;
}

17
pilosa.go
View file

@ -17,6 +17,7 @@ package pilosa
import (
"encoding/json"
"regexp"
"time"
"github.com/pkg/errors"
)
@ -65,6 +66,9 @@ var (
// we won't need this error at all by 2.0 though.
ErrClusterDoesNotOwnShard = errors.New("node does not own shard")
// ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match
ErrPreconditionFailed = errors.New("precondition failed")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
@ -124,6 +128,15 @@ func newNotFoundError(err error) NotFoundError {
return NotFoundError{err}
}
type PreconditionFailedError struct {
error
}
// newPreconditionFailedError returns err wrapped in a PreconditionFailedError.
func newPreconditionFailedError(err error) PreconditionFailedError {
return PreconditionFailedError{err}
}
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
@ -191,6 +204,10 @@ func stringSlicesAreEqual(a, b []string) bool {
return true
}
func timestamp() int64 {
return time.Now().UnixNano()
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {

19
server.go
View file

@ -356,14 +356,11 @@ func NewServer(opts ...ServerOption) (*Server, error) {
}
s.executor = newExecutor(executorOpts...)
// s.holder.translateFile.logger = s.logger
path, err := expandDirName(s.dataDir)
if err != nil {
return nil, err
}
s.holder.Path = path
// s.holder.translateFile.Path = filepath.Join(path, ".keys")
s.holder.Logger = s.logger
s.holder.Stats.SetLogger(s.logger)
@ -716,10 +713,13 @@ func (s *Server) receiveMessage(m Message) error {
}
case *CreateIndexMessage:
opt := obj.Meta
_, err := s.holder.CreateIndex(obj.Index, *opt)
idx, err := s.holder.CreateIndex(obj.Index, *opt)
if err != nil {
return err
}
idx.mu.Lock()
idx.createdAt = obj.CreatedAt
idx.mu.Unlock()
case *DeleteIndexMessage:
if err := s.holder.DeleteIndex(obj.Index); err != nil {
return err
@ -730,10 +730,13 @@ func (s *Server) receiveMessage(m Message) error {
return fmt.Errorf("local index not found: %s", obj.Index)
}
opt := obj.Meta
_, err := idx.createFieldIfNotExists(obj.Field, opt)
fld, err := idx.createFieldIfNotExists(obj.Field, opt)
if err != nil {
return err
}
fld.mu.Lock()
fld.createdAt = obj.CreatedAt
fld.mu.Unlock()
case *DeleteFieldMessage:
idx := s.holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
@ -767,6 +770,12 @@ func (s *Server) receiveMessage(m Message) error {
if err != nil {
return err
}
if !s.isCoordinator {
if obj.Schema != nil {
s.holder.applyCreatedAt(obj.Schema.Indexes)
}
}
case *ResizeInstruction:
err := s.cluster.followResizeInstruction(obj)
if err != nil {

197
server/handler_test.go
View file

@ -195,19 +195,81 @@ func TestHandler_Endpoints(t *testing.T) {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
body := w.Body.String()
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}
`, pilosa.ShardWidth)
body := strings.TrimSpace(w.Body.String())
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
if body != target {
t.Fatalf("%s != %s", target, body)
}
})
t.Run("Import", func(t *testing.T) {
indexInfo := cmd.API.Schema(context.Background())
err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false)
if err != nil {
t.Fatalf("applying schema: %v", err)
}
idx := indexInfo[0]
fld := indexInfo[0].Fields[0]
msg := pilosa.ImportRequest{
Index: idx.Name,
IndexCreatedAt: idx.CreatedAt,
Field: fld.Name,
FieldCreatedAt: fld.CreatedAt,
Shard: 0,
}
ser := proto.Serializer{}
data, err := ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name)
httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, httpReq)
if w.Code != 200 {
t.Fatalf(w.Body.String())
}
msg.IndexCreatedAt = -idx.CreatedAt
msg.FieldCreatedAt = -fld.CreatedAt
data, err = ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
w = httptest.NewRecorder()
h.ServeHTTP(w, httpReq)
if w.Code != 412 {
t.Fatal("expected: Precondition Failed, got:" + w.Body.String())
}
})
t.Run("ImportRoaring", func(t *testing.T) {
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
idx, err := cmd.API.Index(context.Background(), "i0")
if err != nil {
t.Fatal(err)
}
fld, err := cmd.API.Field(context.Background(), "i0", "f1")
if err != nil {
t.Fatal(err)
}
msg := pilosa.ImportRoaringRequest{
Clear: false,
IndexCreatedAt: idx.CreatedAt(),
FieldCreatedAt: fld.CreatedAt(),
Clear: false,
Views: map[string][]byte{
"": roaringData,
},
@ -217,10 +279,14 @@ func TestHandler_Endpoints(t *testing.T) {
if err != nil {
t.Fatal(err)
}
httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, httpReq)
if w.Code != 200 {
t.Fatalf("Unexpected response body: %s", w.Body.String())
}
resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"})
if err != nil {
t.Fatalf("querying: %v", err)
@ -761,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected response body: %s", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Verify index is gone.
if hldr.Index("i") != nil {
@ -779,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
})
@ -960,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create index again
@ -970,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusConflict {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Errorf("unexpected body: %q", w.Body.String())
}
}
// create field
@ -980,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create field again
@ -990,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusConflict {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Errorf("unexpected body: %q", w.Body.String())
}
}
// delete field
@ -1000,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// delete field again
@ -1020,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// delete index again
@ -1030,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusNotFound {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
})
@ -1042,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create field
@ -1052,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// set some bits