scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into create-fragment-error

Browse source
This commit is contained in:
Travis Turner 2018-10-17 18:10:00 -05:00 • committed by GitHub
commit cdcbb6ed61
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 3563 additions and 2055 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.circleci/config.yml
View file

@ -72,6 +72,16 @@ jobs:
- *fast-checkout
- run: sudo pip install awscli
- run: make prerelease-upload
dockerhub-upload:
<<: *defaults
steps:
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
- *fast-checkout
- setup_remote_docker
- run: make docker
- run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
- run: docker login -u $DOCKER_USER -p $DOCKER_PASS
- run: docker push pilosa/pilosa:master
workflows:
version: 2
test:
@ -105,3 +115,7 @@ workflows:
- prerelease-upload:
requires:
- prerelease
- dockerhub-upload:
requires:
- linter
- test-golang-1.10

150
api.go
View file

@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
return QueryResponse{}, errors.Wrap(err, "validating api method")
}
resp := QueryResponse{}
q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
if err != nil {
return resp, errors.Wrap(err, "parsing")
return QueryResponse{}, errors.Wrap(err, "parsing")
}
execOpts := &execOptions{
Remote: req.Remote,
@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
}
results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {
return resp, errors.Wrap(err, "executing")
return QueryResponse{}, errors.Wrap(err, "executing")
}
resp.Results = results
// Fill column attributes if requested.
// execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true)
if execOpts.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Row)
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs)
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if api.holder.translateFile != nil {
for _, col := range resp.ColumnAttrSets {
v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
return resp, nil
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
ax := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if len(attrs) == 0 {
continue
}
// Append column with attributes.
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return ax, nil
}
// CreateIndex makes a new Pilosa index.
func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) {
if err := api.validate(apiCreateIndex); err != nil {
@ -321,13 +263,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
nodes := api.cluster.shardNodes(indexName, shard)
var eg errgroup.Group
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound)
}
// only set fields are supported
if field.Type() != FieldTypeSet {
return NewBadRequestError(errors.New("roaring import is only supported for set fields"))
}
for _, node := range nodes {
node := node
if node.ID == api.server.nodeID {
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound)
}
// must make a copy of data to operate on locally. field.importRoaring changes data
d2 := make([]byte, len(data))
copy(d2, data)
@ -379,6 +327,38 @@ func (api *API) DeleteField(_ context.Context, indexName string, fieldName strin
return nil
}
// DeleteAvailableShard a shard ID from the available shard set cache.
func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName string, shardID uint64) error {
if err := api.validate(apiDeleteAvailableShard); err != nil {
return errors.Wrap(err, "validating api method")
}
// Find field.
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound)
}
// Delete shard from the cache.
if err := field.RemoveAvailableShard(shardID); err != nil {
return errors.Wrap(err, "deleting available shard")
}
// Send the delete shard message to all nodes.
err := api.server.SendSync(
&DeleteAvailableShardMessage{
Index: indexName,
Field: fieldName,
ShardID: shardID,
})
if err != nil {
api.server.logger.Printf("problem sending DeleteAvailableShard message: %s", err)
return errors.Wrap(err, "sending DeleteAvailableShard message")
}
api.holder.Stats.CountWithCustomTags("deleteAvailableShard", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName), fmt.Sprintf("field:%s", fieldName)})
return nil
}
// ExportCSV encodes the fragment designated by the index,field,shard as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error {
@ -966,6 +946,7 @@ const (
apiCreateField
apiCreateIndex
apiDeleteField
apiDeleteAvailableShard
apiDeleteIndex
apiDeleteView
apiExportCSV
@ -1004,23 +985,24 @@ var methodsResizing = map[apiMethod]struct{}{
}
var methodsNormal = map[apiMethod]struct{}{
apiCreateField: {},
apiCreateIndex: {},
apiDeleteField: {},
apiDeleteIndex: {},
apiDeleteView: {},
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiFieldAttrDiff: {},
apiImport: {},
apiImportValue: {},
apiIndex: {},
apiIndexAttrDiff: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},
apiShardNodes: {},
apiViews: {},
apiCreateField: {},
apiCreateIndex: {},
apiDeleteField: {},
apiDeleteAvailableShard: {},
apiDeleteIndex: {},
apiDeleteView: {},
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiFieldAttrDiff: {},
apiImport: {},
apiImportValue: {},
apiIndex: {},
apiIndexAttrDiff: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},
apiShardNodes: {},
apiViews: {},
}

6
cluster.go
View file

@ -1946,6 +1946,12 @@ type DeleteFieldMessage struct {
Field string
}
type DeleteAvailableShardMessage struct {
Index string
Field string
ShardID uint64
}
type CreateViewMessage struct {
Index string
Field string

11
diagnostics.go
View file

@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool {
return false
}
// EnrichWithCPUInfo adds CPU information to the diagnostics payload.
func (d *diagnosticsCollector) EnrichWithCPUInfo() {
d.Set("CPUArch", d.server.systemInfo.CPUArch())
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *diagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.systemInfo.Uptime()
@ -265,6 +270,7 @@ type SystemInfo interface {
MemFree() (uint64, error)
MemTotal() (uint64, error)
MemUsed() (uint64, error)
CPUArch() string
}
// newNopSystemInfo creates a no-op implementation of SystemInfo.
@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) {
func (n *nopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}
// CPUArch returns the CPU architecture, such as amd64
func (n *nopSystemInfo) CPUArch() string {
return ""
}

34
docs/query-language.md
View file

@ -266,6 +266,40 @@ ClearRow(stargazer=1)
This represents removing the relationship between the user with id=1 and all repositories.
#### Store
**Spec:**
```
Store(<ROW_CALL>, <FIELD>=<ROW>)
```
**Description:**
`Store` writes the results of <ROW_CALL> to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
**Result Type:** boolean
Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call.
**Examples:**
Store the contents of stargazer row 1 into stargazer row 2:
```request
Store(Row(stargazer=1), stargazer=2)
```
```response
{"results":[true]}
```
Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20.
```request
Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20)
```
```response
{"results":[true]}
```
### Read Operations
#### Row

24
encoding/proto/proto.go
View file

@ -81,6 +81,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
decodeDeleteFieldMessage(msg, mt)
return nil
case *pilosa.DeleteAvailableShardMessage:
msg := &internal.DeleteAvailableShardMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling DeleteAvailableShardMessage")
}
decodeDeleteAvailableShardMessage(msg, mt)
return nil
case *pilosa.CreateViewMessage:
msg := &internal.CreateViewMessage{}
err := proto.Unmarshal(buf, msg)
@ -250,6 +258,8 @@ func encodeToProto(m pilosa.Message) proto.Message {
return encodeCreateFieldMessage(mt)
case *pilosa.DeleteFieldMessage:
return encodeDeleteFieldMessage(mt)
case *pilosa.DeleteAvailableShardMessage:
return encodeDeleteAvailableShardMessage(mt)
case *pilosa.CreateViewMessage:
return encodeCreateViewMessage(mt)
case *pilosa.DeleteViewMessage:
@ -549,6 +559,14 @@ func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFiel
}
}
func encodeDeleteAvailableShardMessage(m *pilosa.DeleteAvailableShardMessage) *internal.DeleteAvailableShardMessage {
return &internal.DeleteAvailableShardMessage{
Index: m.Index,
Field: m.Field,
ShardID: m.ShardID,
}
}
func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage {
return &internal.CreateViewMessage{
Index: m.Index,
@ -775,6 +793,12 @@ func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteF
m.Field = pb.Field
}
func decodeDeleteAvailableShardMessage(pb *internal.DeleteAvailableShardMessage, m *pilosa.DeleteAvailableShardMessage) {
m.Index = pb.Index
m.Field = pb.Field
m.ShardID = pb.ShardID
}
func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) {
m.Index = pb.Index
m.Field = pb.Field

172
executor.go
View file

@ -79,20 +79,21 @@ func newExecutor(opts ...executorOption) *executor {
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
resp := QueryResponse{}
// Verify that an index is set.
if index == "" {
return nil, ErrIndexRequired
return resp, ErrIndexRequired
}
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
return resp, ErrIndexNotFound
}
// Verify that the number of writes do not exceed the maximum.
if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest {
return nil, ErrTooManyWrites
return resp, ErrTooManyWrites
}
// Default options.
@ -105,14 +106,48 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
if !opt.Remote {
for i := range q.Calls {
if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
return nil, err
return resp, err
}
}
}
results, err := e.execute(ctx, index, q, shards, opt)
if err != nil {
return nil, err
return resp, err
}
resp.Results = results
// Fill column attributes if requested.
if opt.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Row)
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs)
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if idx.Keys() {
for _, col := range columnAttrSets {
v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
// Translate response objects from ids to keys, if necessary.
@ -121,11 +156,35 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
for i := range results {
results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
if err != nil {
return nil, err
return resp, err
}
}
}
return results, nil
return resp, nil
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
ax := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if len(attrs) == 0 {
continue
}
// Append column with attributes.
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return ax, nil
}
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
@ -184,6 +243,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeClearBit(ctx, index, c, opt)
case "ClearRow":
return e.executeClearRow(ctx, index, c, shards, opt)
case "Store":
return e.executeSetRow(ctx, index, c, shards, opt)
case "Count":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, shards, opt)
@ -1213,6 +1274,94 @@ func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.
return changed, nil
}
// executeSetRow executes a SetRow() call.
func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
// Ensure the field type supports SetRow().
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("SetRow() argument required: field")
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
if field.Type() != FieldTypeSet {
return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type())
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeSetRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
val := v.(bool)
if prev == nil {
return val
}
return val || prev.(bool)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
return result.(bool), err
}
// executeSetRowShard executes a SetRow() call for a single shard.
func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("SetRow() argument required: field")
}
// Read fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading SetRow() row: %v", err)
} else if !ok {
return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel)
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
// Retrieve source row.
var src *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return false, errors.Wrap(err, "getting source row")
}
src = row
} else {
return false, errors.New("SetRow() requires a source row")
}
// Set the row on the standard view.
changed := false
fragment := e.Holder.fragment(index, fieldName, viewStandard, shard)
if fragment == nil {
// Since the destination fragment doesn't exist, create one.
view, err := field.createViewIfNotExists(viewStandard)
if err != nil {
return false, errors.Wrap(err, "creating view")
}
fragment, err = view.createFragmentIfNotExists(shard)
if err != nil {
return false, errors.Wrapf(err, "creating fragment: %d", shard)
}
}
set, err := fragment.setRow(src, rowID)
if err != nil {
return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard)
}
changed = changed || set
return changed, nil
}
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
// Read colID.
@ -1708,16 +1857,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
var colKey, rowKey, fieldName string
if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" {
switch c.Name {
case "Set", "Clear", "Row", "Range", "SetColumnAttrs":
// Positional args in new PQL syntax require special handling here.
colKey = "_" + columnLabel
fieldName, _ = c.FieldArg()
rowKey = fieldName
} else if c.Name == "SetRowAttrs" {
case "SetRowAttrs":
// Positional args in new PQL syntax require special handling here.
rowKey = "_" + rowLabel
fieldName = callArgString(c, "_field")
} else {
default:
colKey = "col"
fieldName = callArgString(c, "field")
rowKey = "row"

1817
executor_test.go
View file

File diff suppressed because it is too large Load diff

111
field.go
View file

@ -15,6 +15,7 @@
package pilosa
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
@ -230,11 +231,10 @@ func (f *Field) AvailableShards() *roaring.Bitmap {
return b
}
// addRemoteAvailableShards merges the set of available shards into the current known set
// AddRemoteAvailableShards merges the set of available shards into the current known set
// and saves the set to a file.
func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error {
func (f *Field) AddRemoteAvailableShards(b *roaring.Bitmap) error {
f.mergeRemoteAvailableShards(b)
// Save the updated bitmap to the data store.
return f.saveAvailableShards()
}
@ -246,6 +246,70 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
f.remoteAvailableShards = f.remoteAvailableShards.Union(b)
}
// loadAvailableShards reads remoteAvailableShards data for the field, if any.
func (f *Field) loadAvailableShards() error {
bm := roaring.NewBitmap()
// Read data from meta file.
path := filepath.Join(f.path, ".available.shards")
buf, err := ioutil.ReadFile(path)
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading available shards")
} else {
if err := bm.UnmarshalBinary(buf); err != nil {
return errors.Wrap(err, "unmarshaling")
}
}
// Merge bitmap from file into field.
f.mergeRemoteAvailableShards(bm)
return nil
}
// saveAvailableShards writes remoteAvailableShards data for the field.
func (f *Field) saveAvailableShards() error {
f.mu.Lock()
defer f.mu.Unlock()
return f.unprotectedSaveAvailableShards()
}
func (f *Field) unprotectedSaveAvailableShards() error {
// Open or create file.
path := filepath.Join(f.path, ".available.shards")
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return errors.Wrap(err, "opening available shards file")
}
defer file.Close()
// Write available shards to file.
bw := bufio.NewWriter(file)
if _, err = f.remoteAvailableShards.WriteTo(bw); err != nil {
return errors.Wrap(err, "writing bitmap to buffer")
}
bw.Flush()
return nil
}
// RemoveAvailableShard removes a shard from the bitmap cache.
//
// NOTE: This can be overridden on the next sync so all nodes should be updated.
func (f *Field) RemoveAvailableShard(v uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
b := f.remoteAvailableShards.Clone()
if _, err := b.Remove(v); err != nil {
return err
}
f.remoteAvailableShards = b
return f.unprotectedSaveAvailableShards()
}
// Type returns the field type.
func (f *Field) Type() string {
f.mu.RLock()
@ -480,47 +544,6 @@ func (f *Field) applyOptions(opt FieldOptions) error {
return nil
}
// loadAvailableShards reads remoteAvailableShards data for the field, if any.
func (f *Field) loadAvailableShards() error {
bm := roaring.NewBitmap()
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading available shards")
} else {
if err := bm.UnmarshalBinary(buf); err != nil {
return errors.Wrap(err, "unmarshaling")
}
}
// Merge bitmap from file into field.
f.mergeRemoteAvailableShards(bm)
return nil
}
// saveAvailableShards writes remoteAvailableShards data for the field.
func (f *Field) saveAvailableShards() error {
// Open or create file.
file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
return errors.Wrap(err, "opening available shards file")
}
f.mu.RLock()
defer f.mu.RUnlock()
// Write available shards to file.
if _, err := f.remoteAvailableShards.WriteTo(file); err != nil {
return errors.Wrap(err, "writing bitmap to buffer")
}
return nil
}
// Close closes the field and its views.
func (f *Field) Close() error {
f.mu.Lock()

43
field_internal_test.go
View file

@ -349,7 +349,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
if err := f.addRemoteAvailableShards(bm); err != nil {
if err := f.AddRemoteAvailableShards(bm); err != nil {
t.Fatal(err)
}
@ -361,3 +361,44 @@ func TestField_PersistAvailableShards(t *testing.T) {
}
}
// Ensure that persisting available shards having a smaller footprint (for example,
// when going from a bitmap to a smaller, RLE representation) succeeds.
func TestField_PersistAvailableShardsFootprint(t *testing.T) {
f := MustOpenField(OptFieldTypeDefault())
// bm represents remote available shards.
bm := roaring.NewBitmap()
for i := uint64(0); i < 1204; i += 2 {
bm.Add(i)
}
if err := f.AddRemoteAvailableShards(bm); err != nil {
t.Fatal(err)
}
// Reload field and verify that shard data is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
}
bm1 := roaring.NewBitmap()
for i := uint64(1); i < 1204; i += 2 {
bm1.Add(i)
}
if err := f.AddRemoteAvailableShards(bm1); err != nil {
t.Fatal(err)
}
// Reload field and verify that shard data is persisted.
result := bm.Union(bm1)
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), result.Slice()) {
t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
}
}

38
field_test.go
View file

@ -18,7 +18,9 @@ import (
"io/ioutil"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/test"
)
@ -185,3 +187,39 @@ func TestField_NameValidation(t *testing.T) {
}
}
}
// Ensure can update and delete available shards.
func TestField_AvailableShards(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
// Set values on shards 0 & 2, and verify.
if _, err := f.SetBit(0, 100, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, ShardWidth*2, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
t.Fatal(diff)
}
// Set remote shards and verify.
f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4))
if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" {
t.Fatal(diff)
}
// Delete shards; only local shards should remain.
f.RemoveAvailableShard(0)
f.RemoveAvailableShard(1)
f.RemoveAvailableShard(2)
f.RemoveAvailableShard(3)
f.RemoveAvailableShard(4)
if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
t.Fatal(diff)
}
}

54
fragment.go
View file

@ -352,11 +352,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row {
}
// Only use a subset of the containers.
// NOTE: The start & end ranges must be divisible by
// NOTE: The start & end ranges must be divisible by container width.
data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
// Reference bitmap subrange in storage.
// We Clone() data because otherwise row will contains pointers to containers in storage.
// We Clone() data because otherwise row will contain pointers to containers in storage.
// This causes unexpected results when we cache the row and try to use it later.
row := &Row{
segments: []rowSegment{{
@ -491,6 +491,56 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
return changed, nil
}
// setRow replaces an existing row (specified by rowID) with the given
// Row. This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.unprotectedSetRow(row, rowID)
}
func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) {
// TODO: In order to return `changed`, we need to first compare
// the existing row with the given row. Determine if the overhead
// of this is worth having `changed`.
// For now we will assume changed is always true.
changed = true
// First container of the row in storage.
headContainerKey := rowID << shardVsContainerExponent
// Remove every existing container in the row.
for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
f.storage.Containers.Remove(headContainerKey + i)
}
// From the given row, get the rowSegment for this shard.
seg := row.segment(f.shard)
if seg == nil {
return changed, nil
}
// Put each container from rowSegment to fragment storage.
citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent)
for citer.Next() {
k, c := citer.Value()
f.storage.Containers.Put(headContainerKey+(k%(1<<shardVsContainerExponent)), c)
}
// Update the row in cache.
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
f.cache.BulkAdd(rowID, n)
// Snapshot storage.
if err := f.snapshot(); err != nil {
return false, errors.Wrap(err, "snapshotting")
}
f.stats.Count("setRow", 1, 1.0)
return changed, nil
}
// ClearRow clears a row for a given rowID within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) clearRow(rowID uint64) (bool, error) {

48
fragment_internal_test.go
View file

@ -122,6 +122,54 @@ func TestFragment_ClearRow(t *testing.T) {
}
}
// Ensure a fragment can set a row.
func TestFragment_SetRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 7, "")
defer f.Close()
rowID := uint64(1000)
// Set bits on the fragment.
if _, err := f.setBit(rowID, 8000001); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(rowID, 8065536); err != nil {
t.Fatal(err)
}
// Verify data on row.
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
t.Fatalf("unexpected columns: %+v", cols)
}
// Verify count on row.
if n := f.row(rowID).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
}
// Set row (overwrite existing data).
row := NewRow(8000002, 8065537, 8131074)
if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatalf("expected changed value: %v", changed)
}
// Verify data on row.
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
t.Fatalf("unexpected columns after set row: %+v", cols)
}
// Verify count on row.
if n := f.row(rowID).Count(); n != 3 {
t.Fatalf("unexpected count after set row: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.row(rowID).Count(); n != 3 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
// Ensure a fragment can set & read a value.
func TestFragment_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {

7
gopsutil/systeminfo.go
View file

@ -15,6 +15,8 @@
package gopsutil
import (
"runtime"
"github.com/pilosa/pilosa"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
@ -109,6 +111,11 @@ func (s *systemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// CPUArch returns the CPU architecture, such as amd64
func (s *systemInfo) CPUArch() string {
return runtime.GOARCH
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *systemInfo {
return &systemInfo{}

16
gopsutil/systeminfo_test.go
View file

@ -15,7 +15,6 @@
package gopsutil_test
import (
"log"
"testing"
"github.com/pilosa/pilosa"
@ -25,15 +24,6 @@ import (
func TestSystemInfo(t *testing.T) {
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
// Uptime()(uint64, error)
// Platform()(string, error)
// Family()(string, error)
// OSVersion()(string, error)
// KernelVersion()(string, error)
// MemFree()(uint64, error)
// MemTotal()(uint64, error)
// MemUsed()(uint64, error)
//
uptime, err := systemInfo.Uptime()
if err != nil || uptime == 0 {
t.Fatalf("Error collecting uptime (error: %v)", err)
@ -70,8 +60,12 @@ func TestSystemInfo(t *testing.T) {
}
memtotal, err := systemInfo.MemTotal()
log.Println(memtotal)
if err != nil {
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
}
cpuArch := systemInfo.CPUArch()
if cpuArch == "" {
t.Fatalf("Error getting CPU arch.")
}
}

6
holder.go
View file

@ -101,6 +101,9 @@ func NewHolder() *Holder {
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
// Reset closing in case Holder is being reopened.
h.closing = make(chan struct{})
h.setFileLimit()
h.Logger.Printf("open holder path: %s", h.Path)
@ -178,6 +181,9 @@ func (h *Holder) Close() error {
}
}
// Reset opened in case Holder needs to be reopened.
h.opened = make(chan struct{})
return nil
}

14
holder_internal_test.go
View file

@ -209,8 +209,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7)
// Set highest shard.
hldr0.Field("i", "f").addRemoteAvailableShards(roaring.NewBitmap(0, 1))
hldr0.Field("y", "z").addRemoteAvailableShards(roaring.NewBitmap(0, 1, 2))
hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1))
hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2))
// Keep replication the same and ensure we get the expected results.
cluster.ReplicaN = 2
@ -287,3 +287,13 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
}
}
}
// Ensure holder can reopen.
func TestHolderCleaner_Reopen(t *testing.T) {
h := NewHolder()
h.Path = "path"
h.Open()
h.Close()
h.Open()
h.Close()
}

115
http/handler.go
View file

@ -170,13 +170,34 @@ func (h *Handler) Close() error {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["Home"] = queryValidationSpecRequired()
h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired()
h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired()
h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired()
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetIndexes"] = queryValidationSpecRequired()
h.validators["GetIndex"] = queryValidationSpecRequired()
h.validators["PostIndex"] = queryValidationSpecRequired()
h.validators["DeleteIndex"] = queryValidationSpecRequired()
h.validators["PostField"] = queryValidationSpecRequired()
h.validators["DeleteField"] = queryValidationSpecRequired()
h.validators["PostImport"] = queryValidationSpecRequired()
h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetInfo"] = queryValidationSpecRequired()
h.validators["RecalculateCaches"] = queryValidationSpecRequired()
h.validators["GetSchema"] = queryValidationSpecRequired()
h.validators["GetStatus"] = queryValidationSpecRequired()
h.validators["GetVersion"] = queryValidationSpecRequired()
h.validators["PostClusterMessage"] = queryValidationSpecRequired()
h.validators["GetFragmentBlockData"] = queryValidationSpecRequired()
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired()
h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
h.validators["GetNodes"] = queryValidationSpecRequired()
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["GetTranslateData"] = queryValidationSpecRequired("offset")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -203,40 +224,41 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
// newRouter creates a new mux http router.
func newRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
router.HandleFunc("/", handler.handleHome).Methods("GET")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
// /internal endpoints are for internal use only; they may change at any time.
// DO NOT rely on these for external applications!
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE")
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
// TODO: Apply MethodNotAllowed statuses to all endpoints.
// Ideally this would be automatic, as described in this (wontfix) ticket:
@ -521,7 +543,12 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
return err
}
// Unmarshal expected values.
var _p _postIndexRequest
_p := _postIndexRequest{
Options: pilosa.IndexOptions{
Keys: false,
TrackExistence: true,
},
}
if err := json.Unmarshal(b, &_p); err != nil {
return errors.Wrap(err, "unmarshalling expected values")
}
@ -597,7 +624,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp := successResponse{}
// Decode request.
var req postIndexRequest
req := postIndexRequest{
Options: pilosa.IndexOptions{
Keys: false,
TrackExistence: true,
},
}
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil && err != io.EOF {
resp.write(w, err)
@ -815,6 +847,22 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
}
// handleDeleteRemoteAvailableShard handles DELETE /field/{field}/available-shards/{shardID} request.
func (h *Handler) handleDeleteRemoteAvailableShard(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
shardID, _ := strconv.ParseUint(mux.Vars(r)["shardID"], 10, 64)
resp := successResponse{}
err := h.api.DeleteAvailableShard(r.Context(), indexName, fieldName, shardID)
resp.write(w, err)
}
// handlePostFieldAttrDiff handles POST /internal/field/attr/diff requests.
func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -1473,11 +1521,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
return
}
resp := &pilosa.ImportResponse{}
// TODO give meaningful stats for import
err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body)
resp := &pilosa.ImportResponse{}
if err != nil {
resp.Err = err.Error()
if _, ok := err.(pilosa.BadRequestError); ok {
w.WriteHeader(http.StatusBadRequest)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
}
// Marshal response object.
buf, err := h.api.Serializer.Marshal(resp)

7
http/handler_internal_test.go
View file

@ -31,8 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
expected postIndexRequest
err string
}{
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}},
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}},
{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
@ -53,7 +54,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
if test.err == "" {
if !reflect.DeepEqual(*actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, *actual)
t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json)
}
}

7
index.go
View file

@ -69,9 +69,10 @@ func NewIndex(path, name string) (*Index, error) {
newAttrStore: newNopAttrStore,
columnAttrs: nopStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
logger: NopLogger,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
logger: NopLogger,
trackExistence: true,
}, nil
}

398
internal/private.pb.go
View file

@ -20,6 +20,7 @@
CreateIndexMessage
CreateFieldMessage
DeleteFieldMessage
DeleteAvailableShardMessage
Field
Schema
Index
@ -397,6 +398,40 @@ func (m *DeleteFieldMessage) GetField() string {
return ""
}
type DeleteAvailableShardMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
ShardID uint64 `protobuf:"varint,3,opt,name=ShardID,proto3" json:"ShardID,omitempty"`
}
func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShardMessage{} }
func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteAvailableShardMessage) ProtoMessage() {}
func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptorPrivate, []int{12}
}
func (m *DeleteAvailableShardMessage) GetIndex() string {
if m != nil {
return m.Index
}
return ""
}
func (m *DeleteAvailableShardMessage) GetField() string {
if m != nil {
return m.Field
}
return ""
}
func (m *DeleteAvailableShardMessage) GetShardID() uint64 {
if m != nil {
return m.ShardID
}
return 0
}
type Field struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
@ -406,7 +441,7 @@ type Field struct {
func (m *Field) Reset() { *m = Field{} }
func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} }
func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} }
func (m *Field) GetName() string {
if m != nil {
@ -436,7 +471,7 @@ type Schema struct {
func (m *Schema) Reset() { *m = Schema{} }
func (m *Schema) String() string { return proto.CompactTextString(m) }
func (*Schema) ProtoMessage() {}
func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} }
func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} }
func (m *Schema) GetIndexes() []*Index {
if m != nil {
@ -453,7 +488,7 @@ type Index struct {
func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} }
func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} }
func (m *Index) GetName() string {
if m != nil {
@ -478,7 +513,7 @@ type URI struct {
func (m *URI) Reset() { *m = URI{} }
func (m *URI) String() string { return proto.CompactTextString(m) }
func (*URI) ProtoMessage() {}
func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} }
func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} }
func (m *URI) GetScheme() string {
if m != nil {
@ -510,7 +545,7 @@ type Node struct {
func (m *Node) Reset() { *m = Node{} }
func (m *Node) String() string { return proto.CompactTextString(m) }
func (*Node) ProtoMessage() {}
func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} }
func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} }
func (m *Node) GetID() string {
if m != nil {
@ -541,7 +576,7 @@ type NodeStateMessage struct {
func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} }
func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) }
func (*NodeStateMessage) ProtoMessage() {}
func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} }
func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} }
func (m *NodeStateMessage) GetNodeID() string {
if m != nil {
@ -565,7 +600,7 @@ type NodeEventMessage struct {
func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} }
func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) }
func (*NodeEventMessage) ProtoMessage() {}
func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} }
func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} }
func (m *NodeEventMessage) GetEvent() uint32 {
if m != nil {
@ -590,7 +625,7 @@ type NodeStatus struct {
func (m *NodeStatus) Reset() { *m = NodeStatus{} }
func (m *NodeStatus) String() string { return proto.CompactTextString(m) }
func (*NodeStatus) ProtoMessage() {}
func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} }
func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} }
func (m *NodeStatus) GetNode() *Node {
if m != nil {
@ -621,7 +656,7 @@ type IndexStatus struct {
func (m *IndexStatus) Reset() { *m = IndexStatus{} }
func (m *IndexStatus) String() string { return proto.CompactTextString(m) }
func (*IndexStatus) ProtoMessage() {}
func (*IndexStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} }
func (*IndexStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} }
func (m *IndexStatus) GetName() string {
if m != nil {
@ -645,7 +680,7 @@ type FieldStatus struct {
func (m *FieldStatus) Reset() { *m = FieldStatus{} }
func (m *FieldStatus) String() string { return proto.CompactTextString(m) }
func (*FieldStatus) ProtoMessage() {}
func (*FieldStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} }
func (*FieldStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} }
func (m *FieldStatus) GetName() string {
if m != nil {
@ -670,7 +705,7 @@ type ClusterStatus struct {
func (m *ClusterStatus) Reset() { *m = ClusterStatus{} }
func (m *ClusterStatus) String() string { return proto.CompactTextString(m) }
func (*ClusterStatus) ProtoMessage() {}
func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} }
func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} }
func (m *ClusterStatus) GetClusterID() string {
if m != nil {
@ -703,7 +738,7 @@ type BSIGroup struct {
func (m *BSIGroup) Reset() { *m = BSIGroup{} }
func (m *BSIGroup) String() string { return proto.CompactTextString(m) }
func (*BSIGroup) ProtoMessage() {}
func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} }
func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} }
func (m *BSIGroup) GetName() string {
if m != nil {
@ -742,7 +777,7 @@ type CreateViewMessage struct {
func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} }
func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) }
func (*CreateViewMessage) ProtoMessage() {}
func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} }
func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} }
func (m *CreateViewMessage) GetIndex() string {
if m != nil {
@ -774,7 +809,7 @@ type DeleteViewMessage struct {
func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} }
func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteViewMessage) ProtoMessage() {}
func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} }
func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} }
func (m *DeleteViewMessage) GetIndex() string {
if m != nil {
@ -809,7 +844,7 @@ type ResizeInstruction struct {
func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} }
func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) }
func (*ResizeInstruction) ProtoMessage() {}
func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} }
func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} }
func (m *ResizeInstruction) GetJobID() int64 {
if m != nil {
@ -864,7 +899,7 @@ type ResizeSource struct {
func (m *ResizeSource) Reset() { *m = ResizeSource{} }
func (m *ResizeSource) String() string { return proto.CompactTextString(m) }
func (*ResizeSource) ProtoMessage() {}
func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} }
func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} }
func (m *ResizeSource) GetNode() *Node {
if m != nil {
@ -911,7 +946,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp
func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) }
func (*ResizeInstructionComplete) ProtoMessage() {}
func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) {
return fileDescriptorPrivate, []int{28}
return fileDescriptorPrivate, []int{29}
}
func (m *ResizeInstructionComplete) GetJobID() int64 {
@ -942,7 +977,7 @@ type SetCoordinatorMessage struct {
func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} }
func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*SetCoordinatorMessage) ProtoMessage() {}
func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} }
func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} }
func (m *SetCoordinatorMessage) GetNew() *Node {
if m != nil {
@ -958,7 +993,7 @@ type UpdateCoordinatorMessage struct {
func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} }
func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*UpdateCoordinatorMessage) ProtoMessage() {}
func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} }
func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} }
func (m *UpdateCoordinatorMessage) GetNew() *Node {
if m != nil {
@ -975,7 +1010,7 @@ type Topology struct {
func (m *Topology) Reset() { *m = Topology{} }
func (m *Topology) String() string { return proto.CompactTextString(m) }
func (*Topology) ProtoMessage() {}
func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} }
func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} }
func (m *Topology) GetClusterID() string {
if m != nil {
@ -997,7 +1032,7 @@ type RecalculateCaches struct {
func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} }
func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) }
func (*RecalculateCaches) ProtoMessage() {}
func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} }
func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{33} }
func init() {
proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta")
@ -1012,6 +1047,7 @@ func init() {
proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage")
proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage")
proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage")
proto.RegisterType((*DeleteAvailableShardMessage)(nil), "internal.DeleteAvailableShardMessage")
proto.RegisterType((*Field)(nil), "internal.Field")
proto.RegisterType((*Schema)(nil), "internal.Schema")
proto.RegisterType((*Index)(nil), "internal.Index")
@ -1487,6 +1523,41 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *DeleteAvailableShardMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DeleteAvailableShardMessage) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Field) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.ShardID != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID))
}
return i, nil
}
func (m *Field) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@ -2508,6 +2579,23 @@ func (m *DeleteFieldMessage) Size() (n int) {
return n
}
func (m *DeleteAvailableShardMessage) Size() (n int) {
var l int
_ = l
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.ShardID != 0 {
n += 1 + sovPrivate(uint64(m.ShardID))
}
return n
}
func (m *Field) Size() (n int) {
var l int
_ = l
@ -4442,6 +4530,133 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DeleteAvailableShardMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DeleteAvailableShardMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ShardID", wireType)
}
m.ShardID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.ShardID |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Field) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@ -7184,74 +7399,75 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1095 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44,
0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0xdb, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x15,
0x0d, 0x95, 0x08, 0x55, 0x7b, 0xc3, 0x5f, 0xa5, 0x92, 0x6c, 0x28, 0xa6, 0x24, 0x94, 0x71, 0x92,
0xbb, 0x5e, 0x4c, 0x76, 0x47, 0x8d, 0x15, 0xaf, 0xc7, 0xd8, 0xe3, 0x24, 0xdb, 0x0b, 0x6e, 0x41,
0xe2, 0x05, 0x10, 0x4f, 0xc2, 0x23, 0x70, 0xc9, 0x23, 0xa0, 0xf0, 0x22, 0x68, 0xce, 0x8c, 0x7f,
0xb2, 0xd9, 0xb0, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f,
0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x46, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c,
0x26, 0x4f, 0xa1, 0x1b, 0x24, 0x23, 0x7e, 0xb6, 0xc3, 0x25, 0xf3, 0x3c, 0x70, 0x9f, 0xf1, 0x49,
0xee, 0x3b, 0x6b, 0xd6, 0x7a, 0x87, 0xe2, 0xd9, 0xfb, 0x10, 0x16, 0xf7, 0x32, 0x36, 0x3c, 0xde,
0x3e, 0x8b, 0x72, 0xc9, 0x93, 0x21, 0xf7, 0x5d, 0xd4, 0x4e, 0xa1, 0xe4, 0x0f, 0x0b, 0x6e, 0x7c,
0x1d, 0xf1, 0x78, 0xf4, 0x7d, 0x2a, 0x23, 0x91, 0xe4, 0xde, 0x7b, 0xd0, 0xdd, 0x62, 0xc3, 0x23,
0xbe, 0x37, 0x49, 0x39, 0x7a, 0xec, 0xd2, 0x1a, 0xa8, 0xb4, 0x61, 0xf4, 0x4a, 0x7b, 0xec, 0xd3,
0x1a, 0xf0, 0xd6, 0xa0, 0xb7, 0x17, 0x8d, 0xf9, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf6, 0x5b, 0x78,
0xbb, 0x09, 0x29, 0xaa, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0x4b, 0xe0, 0xec, 0x44, 0x89, 0xdf,
0x5d, 0xb3, 0xd6, 0x1d, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0x27, 0xf6,
0xea, 0x27, 0x12, 0x02, 0x8b, 0xc1, 0x38, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7,
0xed, 0x2c, 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xda, 0x8c, 0xc5, 0xf0, 0x78, 0xc0,
0x24, 0xa3, 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x3b, 0x63, 0xa7, 0x05, 0x85,
0x62, 0x1e, 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86,
0x47, 0x2c, 0x1b, 0x61, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78,
0x26, 0x01, 0x2c, 0x37, 0xe2, 0x1b, 0x9a, 0x2b, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe4, 0xbe, 0xb5,
0xe6, 0xac, 0xbb, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0xe3, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00,
0xb9, 0x03, 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8,
0x19, 0xd2, 0xc8, 0xbd, 0xc7, 0xd0, 0x09, 0x25, 0x4b, 0x46, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfc,
0x60, 0xa3, 0x6c, 0x9c, 0x8d, 0xca, 0x6c, 0xa3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa1, 0xd5, 0x95,
0x77, 0xbf, 0x80, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0x27, 0x65, 0x56, 0x8f, 0xf9, 0x44, 0xbd,
0xff, 0x84, 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xdb, 0x9f, 0x5a, 0xe4, 0x00, 0xbc,
0xad, 0x8c, 0x33, 0xc9, 0x31, 0xc8, 0x0e, 0xcf, 0x73, 0xf6, 0x92, 0x5f, 0x9d, 0x71, 0x9d, 0x45,
0xbb, 0x99, 0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x3e, 0x78, 0x03, 0x1e, 0x73, 0xc9, 0x4d,
0xd7, 0xff, 0x87, 0x5f, 0x12, 0x96, 0x1c, 0xe6, 0xdb, 0x7a, 0xf7, 0xc0, 0x55, 0x23, 0x84, 0x14,
0x7a, 0x0f, 0x6f, 0xd5, 0x79, 0xaa, 0xa6, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0x67, 0xee,
0xc3, 0x66, 0xb4, 0xd2, 0x7d, 0x13, 0xca, 0xc1, 0x50, 0x2b, 0x75, 0xa8, 0xe6, 0xf8, 0x99, 0x68,
0x4f, 0xca, 0xe7, 0x5e, 0x37, 0x1a, 0x79, 0x61, 0x50, 0xd5, 0x95, 0xbb, 0x6c, 0xcc, 0xcd, 0x1d,
0x3c, 0x57, 0x54, 0xec, 0xf9, 0x54, 0x94, 0x7b, 0xd5, 0xc9, 0x6a, 0xbb, 0x38, 0xca, 0x3d, 0x0a,
0xe4, 0x11, 0xb4, 0xc3, 0xe1, 0x11, 0x1f, 0x33, 0xef, 0x23, 0x58, 0x40, 0x1e, 0x3c, 0x37, 0xcd,
0x76, 0x73, 0x2a, 0x89, 0xb4, 0xd4, 0x93, 0x81, 0xe1, 0x3f, 0x93, 0xd3, 0x3d, 0x68, 0x63, 0xf4,
0xdc, 0x77, 0xa7, 0xdd, 0x20, 0x4e, 0x8d, 0x9a, 0x6c, 0x83, 0xb3, 0x4f, 0x03, 0x35, 0x44, 0xc8,
0xa0, 0xf4, 0x62, 0x24, 0xe5, 0xfb, 0x1b, 0x91, 0x4b, 0x93, 0x0d, 0x3c, 0x2b, 0xec, 0xb9, 0xc8,
0x24, 0xa6, 0xbe, 0x4f, 0xf1, 0x4c, 0x5e, 0x80, 0xbb, 0x2b, 0x46, 0xdc, 0x5b, 0x04, 0x3b, 0x18,
0x18, 0x1f, 0x76, 0x30, 0xf0, 0xde, 0x47, 0xf7, 0x26, 0x35, 0xfd, 0x9a, 0xc4, 0x3e, 0x0d, 0x28,
0x06, 0xbe, 0x0b, 0xfd, 0x20, 0xdf, 0x12, 0x22, 0x1b, 0x45, 0x09, 0x93, 0x22, 0x33, 0x6b, 0xf7,
0x22, 0x48, 0x9e, 0xc0, 0x92, 0x72, 0x1f, 0x4a, 0x26, 0x79, 0x59, 0xbf, 0x15, 0x68, 0x2b, 0xac,
0x0a, 0x67, 0x24, 0x1c, 0x04, 0x65, 0x57, 0x56, 0x10, 0x05, 0xf2, 0x9d, 0xf6, 0xb0, 0x7d, 0xc2,
0x13, 0xd9, 0xe8, 0x00, 0x94, 0xd1, 0x41, 0x9f, 0x6a, 0xc1, 0x23, 0xfa, 0x29, 0x86, 0xf3, 0x62,
0xcd, 0x59, 0xa1, 0x14, 0x75, 0xe4, 0x57, 0x0b, 0xa0, 0x24, 0x54, 0xe4, 0xd5, 0x15, 0xeb, 0xea,
0x2b, 0xde, 0x7a, 0x59, 0x63, 0xd3, 0xb2, 0x4b, 0xb5, 0x95, 0xc6, 0x69, 0xd9, 0x03, 0x9f, 0xd4,
0x3d, 0xa0, 0x8b, 0x77, 0x7b, 0xaa, 0x07, 0x74, 0xd4, 0xba, 0x13, 0x9e, 0x43, 0xaf, 0x81, 0xcf,
0xec, 0x87, 0x8f, 0xab, 0x7e, 0xb0, 0xa7, 0x5d, 0x22, 0x6e, 0x5c, 0x96, 0x5d, 0xf1, 0x0c, 0x7a,
0x0d, 0x78, 0xa6, 0xc7, 0x75, 0xb8, 0xf9, 0xd5, 0x09, 0x8b, 0x62, 0x76, 0x18, 0xeb, 0xf5, 0x54,
0x2e, 0xd9, 0x69, 0x98, 0x44, 0xd0, 0xdf, 0x8a, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59,
0x03, 0x55, 0xf1, 0x6a, 0x60, 0x76, 0xfd, 0xbc, 0xbb, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39,
0xc7, 0x5a, 0x49, 0x0e, 0xa0, 0xb3, 0x19, 0x06, 0x4f, 0x33, 0x51, 0xa4, 0x33, 0x49, 0x97, 0x1f,
0x4c, 0xfb, 0xf2, 0x07, 0xd3, 0xb9, 0xf4, 0xc1, 0x74, 0xab, 0x0f, 0x26, 0x09, 0x61, 0x59, 0xef,
0x2b, 0x35, 0xaf, 0xd7, 0x59, 0x57, 0xe5, 0xd7, 0xcc, 0x69, 0x7c, 0xcd, 0x42, 0x58, 0xd6, 0x6b,
0xe9, 0x4d, 0x3a, 0xfd, 0xdd, 0x86, 0x65, 0xca, 0xf3, 0xe8, 0x15, 0x0f, 0x92, 0x5c, 0x66, 0xc5,
0x50, 0x6d, 0x1f, 0x75, 0xff, 0x5b, 0x71, 0x68, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x4e, 0xa7, 0x7b,
0x0f, 0xa0, 0x37, 0x3d, 0x9d, 0x97, 0x4d, 0x9b, 0x26, 0xde, 0x03, 0x58, 0x08, 0x45, 0x91, 0x0d,
0xab, 0xf6, 0x6d, 0x6c, 0x44, 0xcd, 0x4c, 0xab, 0x69, 0x69, 0xd6, 0x18, 0x8d, 0xd6, 0x9c, 0xd1,
0x78, 0x3c, 0xd5, 0x4a, 0x7e, 0x1b, 0x2f, 0xbc, 0x53, 0x5f, 0xb8, 0xa0, 0xa6, 0x17, 0xad, 0xc9,
0x2f, 0x16, 0xdc, 0x68, 0x52, 0x78, 0xad, 0xc1, 0xad, 0x2a, 0x62, 0xcf, 0xac, 0x88, 0x33, 0xab,
0x22, 0x6e, 0x5d, 0x91, 0xfa, 0xc3, 0xdc, 0x6a, 0x7c, 0x98, 0xc9, 0x31, 0xdc, 0xb9, 0x54, 0xa6,
0x2d, 0x31, 0x4e, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa,
0x05, 0xf2, 0x19, 0xdc, 0x0e, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x35, 0x70, 0x76, 0xf9, 0xe9,
0x15, 0xcf, 0x57, 0x2a, 0xf2, 0x25, 0xf8, 0xfb, 0xe9, 0x88, 0x49, 0x7e, 0xad, 0xdb, 0x9b, 0xd0,
0xd9, 0x13, 0xa9, 0x88, 0xc5, 0xcb, 0xc9, 0x9c, 0xa9, 0xf7, 0x61, 0x41, 0xef, 0x6f, 0xbd, 0x46,
0xba, 0xb4, 0x14, 0xc9, 0x2d, 0xd5, 0xd0, 0x43, 0x16, 0x0f, 0x8b, 0x58, 0xd1, 0x50, 0x3f, 0x6d,
0xf9, 0xe6, 0xd2, 0x9f, 0xe7, 0xab, 0xd6, 0x5f, 0xe7, 0xab, 0xd6, 0xdf, 0xe7, 0xab, 0xd6, 0x6f,
0xff, 0xac, 0xbe, 0x75, 0xd8, 0xc6, 0x9f, 0xfa, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x31,
0x07, 0xf3, 0xac, 0xe5, 0x0b, 0x00, 0x00,
// 1113 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45,
0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1,
0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9,
0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90,
0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76,
0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c,
0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04,
0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7,
0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4,
0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f,
0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6,
0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82,
0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21,
0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d,
0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22,
0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf,
0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28,
0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87,
0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5,
0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82,
0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6,
0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86,
0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30,
0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e,
0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73,
0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96,
0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce,
0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15,
0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f,
0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0,
0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce,
0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d,
0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b,
0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8,
0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1,
0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59,
0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0,
0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36,
0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45,
0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09,
0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a,
0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42,
0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63,
0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c,
0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4,
0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2,
0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c,
0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68,
0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a,
0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a,
0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75,
0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38,
0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0,
0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c,
0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65,
0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8,
0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26,
0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca,
0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e,
0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96,
0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde,
0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c,
0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea,
0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6,
0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde,
0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad,
0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea,
0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a,
0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff,
0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00,
}

6
internal/private.proto
View file

@ -68,6 +68,12 @@ message DeleteFieldMessage {
string Field = 2;
}
message DeleteAvailableShardMessage {
string Index = 1;
string Field = 2;
uint64 ShardID = 3;
}
message Field {
string Name = 1;
FieldOptions Meta = 2;

2
pilosa.go
View file

@ -117,7 +117,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
ID uint64 `json:"id,omitempty"`
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}

1
pql/pql.peg
View file

@ -11,6 +11,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
/ 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
/ 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
/ 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }

2486
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

10
server.go
View file

@ -481,7 +481,7 @@ func (s *Server) receiveMessage(m Message) error {
if f == nil {
return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field)
}
if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
if err := f.AddRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
case *CreateIndexMessage:
@ -509,6 +509,11 @@ func (s *Server) receiveMessage(m Message) error {
if err := idx.DeleteField(obj.Field); err != nil {
return err
}
case *DeleteAvailableShardMessage:
f := s.holder.Field(obj.Index, obj.Field)
if err := f.RemoveAvailableShard(obj.ShardID); err != nil {
return err
}
case *CreateViewMessage:
f := s.holder.Field(obj.Index, obj.Field)
if f == nil {
@ -648,7 +653,7 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name)
continue
}
if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil {
if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
}
@ -675,6 +680,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.nodeID)
s.diagnostics.Set("ClusterID", s.cluster.id)
s.diagnostics.EnrichWithCPUInfo()
s.diagnostics.EnrichWithOSInfo()
// Flush the diagnostics metrics at startup, then on each tick interval

16
server/handler_test.go
View file

@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) {
})
t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) {
// Roaring import into a non-set field should fail.
if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData))
req.Header.Set("Content-Type", "application/x-binary")
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Status", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))

5
server/server.go
View file

@ -364,7 +364,10 @@ func (m *Command) Close() error {
eg.Go(m.gossipMemberSet.Close)
}
if closer, ok := m.logOutput.(io.Closer); ok {
eg.Go(closer.Close)
// If closer is os.Stdout or os.Stderr, don't close it.
if closer != os.Stdout && closer != os.Stderr {
eg.Go(closer.Close)
}
}
err := eg.Wait()
return errors.Wrap(err, "closing everything")