diff --git a/.circleci/config.yml b/.circleci/config.yml index b42b2aa64..bd1ca86cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,6 +72,16 @@ jobs: - *fast-checkout - run: sudo pip install awscli - run: make prerelease-upload + dockerhub-upload: + <<: *defaults + steps: + - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR + - *fast-checkout + - setup_remote_docker + - run: make docker + - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push pilosa/pilosa:master workflows: version: 2 test: @@ -105,3 +115,7 @@ workflows: - prerelease-upload: requires: - prerelease + - dockerhub-upload: + requires: + - linter + - test-golang-1.10 diff --git a/api.go b/api.go index 44ea0a7b0..0b27d19a8 100644 --- a/api.go +++ b/api.go @@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return QueryResponse{}, errors.Wrap(err, "validating api method") } - resp := QueryResponse{} - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - return resp, errors.Wrap(err, "parsing") + return QueryResponse{}, errors.Wrap(err, "parsing") } execOpts := &execOptions{ Remote: req.Remote, @@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. } - results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) + resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { - return resp, errors.Wrap(err, "executing") + return QueryResponse{}, errors.Wrap(err, "executing") } - resp.Results = results - // Fill column attributes if requested. - // execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true) - if execOpts.ColumnAttrs { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Row) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs) - if err != nil { - return resp, errors.Wrap(err, "reading column attrs") - } - - // Translate column attributes, if necessary. - if api.holder.translateFile != nil { - for _, col := range resp.ColumnAttrSets { - v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID) - if err != nil { - return resp, err - } - col.Key, col.ID = v, 0 - } - } - - resp.ColumnAttrSets = columnAttrSets - } return resp, nil } -// readColumnAttrSets returns a list of column attribute objects by id. -func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { - if index == nil { - return nil, nil - } - - ax := make([]*ColumnAttrSet, 0, len(ids)) - for _, id := range ids { - // Read attributes for column. Skip column if empty. - attrs, err := index.ColumnAttrStore().Attrs(id) - if err != nil { - return nil, errors.Wrap(err, "getting attrs") - } else if len(attrs) == 0 { - continue - } - - // Append column with attributes. - ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) - } - - return ax, nil -} - // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { @@ -321,13 +263,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, nodes := api.cluster.shardNodes(indexName, shard) var eg errgroup.Group + field := api.holder.Field(indexName, fieldName) + if field == nil { + return newNotFoundError(ErrFieldNotFound) + } + + // only set fields are supported + if field.Type() != FieldTypeSet { + return NewBadRequestError(errors.New("roaring import is only supported for set fields")) + } + for _, node := range nodes { node := node if node.ID == api.server.nodeID { - field := api.holder.Field(indexName, fieldName) - if field == nil { - return newNotFoundError(ErrFieldNotFound) - } // must make a copy of data to operate on locally. field.importRoaring changes data d2 := make([]byte, len(data)) copy(d2, data) @@ -379,6 +327,38 @@ func (api *API) DeleteField(_ context.Context, indexName string, fieldName strin return nil } +// DeleteAvailableShard a shard ID from the available shard set cache. +func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName string, shardID uint64) error { + if err := api.validate(apiDeleteAvailableShard); err != nil { + return errors.Wrap(err, "validating api method") + } + + // Find field. + field := api.holder.Field(indexName, fieldName) + if field == nil { + return newNotFoundError(ErrFieldNotFound) + } + + // Delete shard from the cache. + if err := field.RemoveAvailableShard(shardID); err != nil { + return errors.Wrap(err, "deleting available shard") + } + + // Send the delete shard message to all nodes. + err := api.server.SendSync( + &DeleteAvailableShardMessage{ + Index: indexName, + Field: fieldName, + ShardID: shardID, + }) + if err != nil { + api.server.logger.Printf("problem sending DeleteAvailableShard message: %s", err) + return errors.Wrap(err, "sending DeleteAvailableShard message") + } + api.holder.Stats.CountWithCustomTags("deleteAvailableShard", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName), fmt.Sprintf("field:%s", fieldName)}) + return nil +} + // ExportCSV encodes the fragment designated by the index,field,shard as // CSV of the form , func (api *API) ExportCSV(_ context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { @@ -966,6 +946,7 @@ const ( apiCreateField apiCreateIndex apiDeleteField + apiDeleteAvailableShard apiDeleteIndex apiDeleteView apiExportCSV @@ -1004,23 +985,24 @@ var methodsResizing = map[apiMethod]struct{}{ } var methodsNormal = map[apiMethod]struct{}{ - apiCreateField: {}, - apiCreateIndex: {}, - apiDeleteField: {}, - apiDeleteIndex: {}, - apiDeleteView: {}, - apiExportCSV: {}, - apiFragmentBlockData: {}, - apiFragmentBlocks: {}, - apiField: {}, - apiFieldAttrDiff: {}, - apiImport: {}, - apiImportValue: {}, - apiIndex: {}, - apiIndexAttrDiff: {}, - apiQuery: {}, - apiRecalculateCaches: {}, - apiRemoveNode: {}, - apiShardNodes: {}, - apiViews: {}, + apiCreateField: {}, + apiCreateIndex: {}, + apiDeleteField: {}, + apiDeleteAvailableShard: {}, + apiDeleteIndex: {}, + apiDeleteView: {}, + apiExportCSV: {}, + apiFragmentBlockData: {}, + apiFragmentBlocks: {}, + apiField: {}, + apiFieldAttrDiff: {}, + apiImport: {}, + apiImportValue: {}, + apiIndex: {}, + apiIndexAttrDiff: {}, + apiQuery: {}, + apiRecalculateCaches: {}, + apiRemoveNode: {}, + apiShardNodes: {}, + apiViews: {}, } diff --git a/cluster.go b/cluster.go index 868feceee..c3bc67b4d 100644 --- a/cluster.go +++ b/cluster.go @@ -1946,6 +1946,12 @@ type DeleteFieldMessage struct { Field string } +type DeleteAvailableShardMessage struct { + Index string + Field string + ShardID uint64 +} + type CreateViewMessage struct { Index string Field string diff --git a/diagnostics.go b/diagnostics.go index cad7725d7..673fb0c7f 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool { return false } +// EnrichWithCPUInfo adds CPU information to the diagnostics payload. +func (d *diagnosticsCollector) EnrichWithCPUInfo() { + d.Set("CPUArch", d.server.systemInfo.CPUArch()) +} + // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *diagnosticsCollector) EnrichWithOSInfo() { uptime, err := d.server.systemInfo.Uptime() @@ -265,6 +270,7 @@ type SystemInfo interface { MemFree() (uint64, error) MemTotal() (uint64, error) MemUsed() (uint64, error) + CPUArch() string } // newNopSystemInfo creates a no-op implementation of SystemInfo. @@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) { func (n *nopSystemInfo) MemUsed() (uint64, error) { return 0, nil } + +// CPUArch returns the CPU architecture, such as amd64 +func (n *nopSystemInfo) CPUArch() string { + return "" +} diff --git a/docs/query-language.md b/docs/query-language.md index c2319cabd..9bc231e6f 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -266,6 +266,40 @@ ClearRow(stargazer=1) This represents removing the relationship between the user with id=1 and all repositories. +#### Store + +**Spec:** + +``` +Store(, =) +``` + +**Description:** + +`Store` writes the results of to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`. + +**Result Type:** boolean + +Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call. + +**Examples:** + +Store the contents of stargazer row 1 into stargazer row 2: +```request +Store(Row(stargazer=1), stargazer=2) +``` +```response +{"results":[true]} +``` + +Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20. +```request +Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20) +``` +```response +{"results":[true]} +``` + ### Read Operations #### Row diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b49b97454..677f08ce0 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -81,6 +81,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeDeleteFieldMessage(msg, mt) return nil + case *pilosa.DeleteAvailableShardMessage: + msg := &internal.DeleteAvailableShardMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteAvailableShardMessage") + } + decodeDeleteAvailableShardMessage(msg, mt) + return nil case *pilosa.CreateViewMessage: msg := &internal.CreateViewMessage{} err := proto.Unmarshal(buf, msg) @@ -250,6 +258,8 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeCreateFieldMessage(mt) case *pilosa.DeleteFieldMessage: return encodeDeleteFieldMessage(mt) + case *pilosa.DeleteAvailableShardMessage: + return encodeDeleteAvailableShardMessage(mt) case *pilosa.CreateViewMessage: return encodeCreateViewMessage(mt) case *pilosa.DeleteViewMessage: @@ -549,6 +559,14 @@ func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFiel } } +func encodeDeleteAvailableShardMessage(m *pilosa.DeleteAvailableShardMessage) *internal.DeleteAvailableShardMessage { + return &internal.DeleteAvailableShardMessage{ + Index: m.Index, + Field: m.Field, + ShardID: m.ShardID, + } +} + func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage { return &internal.CreateViewMessage{ Index: m.Index, @@ -775,6 +793,12 @@ func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteF m.Field = pb.Field } +func decodeDeleteAvailableShardMessage(pb *internal.DeleteAvailableShardMessage, m *pilosa.DeleteAvailableShardMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.ShardID = pb.ShardID +} + func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) { m.Index = pb.Index m.Field = pb.Field diff --git a/executor.go b/executor.go index 8dd4db569..75b4326fe 100644 --- a/executor.go +++ b/executor.go @@ -79,20 +79,21 @@ func newExecutor(opts ...executorOption) *executor { } // Execute executes a PQL query. -func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { +func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { + resp := QueryResponse{} // Verify that an index is set. if index == "" { - return nil, ErrIndexRequired + return resp, ErrIndexRequired } idx := e.Holder.Index(index) if idx == nil { - return nil, ErrIndexNotFound + return resp, ErrIndexNotFound } // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { - return nil, ErrTooManyWrites + return resp, ErrTooManyWrites } // Default options. @@ -105,14 +106,48 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { for i := range q.Calls { if err := e.translateCall(index, idx, q.Calls[i]); err != nil { - return nil, err + return resp, err } } } results, err := e.execute(ctx, index, q, shards, opt) if err != nil { - return nil, err + return resp, err + } + + resp.Results = results + + // Fill column attributes if requested. + if opt.ColumnAttrs { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Row) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs) + if err != nil { + return resp, errors.Wrap(err, "reading column attrs") + } + + // Translate column attributes, if necessary. + if idx.Keys() { + for _, col := range columnAttrSets { + v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + + resp.ColumnAttrSets = columnAttrSets } // Translate response objects from ids to keys, if necessary. @@ -121,11 +156,35 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar for i := range results { results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) if err != nil { - return nil, err + return resp, err } } } - return results, nil + + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, errors.Wrap(err, "getting attrs") + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil } func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { @@ -184,6 +243,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeClearBit(ctx, index, c, opt) case "ClearRow": return e.executeClearRow(ctx, index, c, shards, opt) + case "Store": + return e.executeSetRow(ctx, index, c, shards, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) @@ -1213,6 +1274,94 @@ func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql. return changed, nil } +// executeSetRow executes a SetRow() call. +func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + // Ensure the field type supports SetRow(). + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + if field.Type() != FieldTypeSet { + return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSetRowShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + val := v.(bool) + if prev == nil { + return val + } + return val || prev.(bool) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + return result.(bool), err +} + +// executeSetRowShard executes a SetRow() call for a single shard. +func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + + // Read fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading SetRow() row: %v", err) + } else if !ok { + return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel) + } + + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + // Retrieve source row. + var src *Row + if len(c.Children) == 1 { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return false, errors.Wrap(err, "getting source row") + } + src = row + } else { + return false, errors.New("SetRow() requires a source row") + } + + // Set the row on the standard view. + changed := false + fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) + if fragment == nil { + // Since the destination fragment doesn't exist, create one. + view, err := field.createViewIfNotExists(viewStandard) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + fragment, err = view.createFragmentIfNotExists(shard) + if err != nil { + return false, errors.Wrapf(err, "creating fragment: %d", shard) + } + } + set, err := fragment.setRow(src, rowID) + if err != nil { + return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) + } + changed = changed || set + + return changed, nil +} + // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { // Read colID. @@ -1708,16 +1857,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { var colKey, rowKey, fieldName string - if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + switch c.Name { + case "Set", "Clear", "Row", "Range", "SetColumnAttrs": // Positional args in new PQL syntax require special handling here. colKey = "_" + columnLabel fieldName, _ = c.FieldArg() rowKey = fieldName - } else if c.Name == "SetRowAttrs" { + case "SetRowAttrs": // Positional args in new PQL syntax require special handling here. rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") - } else { + default: colKey = "col" fieldName = callArgString(c, "field") rowKey = "row" diff --git a/executor_test.go b/executor_test.go index d6d529efe..f2aab6fff 100644 --- a/executor_test.go +++ b/executor_test.go @@ -35,106 +35,76 @@ import ( // Ensure a row query can be executed. func TestExecutor_Execute_Row(t *testing.T) { - t.Run("Row", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20) + + `SetRowAttrs(f, 10, foo="bar", baz=123)` + + `Set(1000, f=100)` + + `SetColumnAttrs(1000, foo="bar", baz=123)` + readQueries := []string{ + `Row(f=10)`, + `Options(Row(f=10), excludeColumns=true)`, + `Options(Row(f=10), excludeRowAttrs=true)`, } - if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit column attributes. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeColumns: true}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := responses[1].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit row attributes. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeRowAttrs: true}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[2].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) - t.Run("Column", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { - t.Fatal(err) + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one-hundred", f=1) + Set("two-hundred", f=1)` + readQueries := []string{`Row(f=1)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) } }) - t.Run("Keys", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { - t.Fatal(err) + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(100, f="one") + Set(200, f="one")` + readQueries := []string{`Row(f="one")`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100, 200}) { + t.Fatalf("unexpected columns: %+v", columns) } + }) - _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ - Index: "i", - Query: `` + - `Set("foo", f="bar")` + "\n" + - `Set("foo", f="baz")` + "\n" + - `Set("bat", f="bar")` + "\n" + - `Set("aaa", f="bbb")` + "\n", - }) - if err != nil { - t.Fatalf("querying: %v", err) - } - - if results, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ - Index: "i", - Query: `Row(f="bar")`, - }); err != nil { - t.Fatal(err) - } else if diff := cmp.Diff(results.Results, []interface{}{ + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := `` + + `Set("foo", f="bar")` + "\n" + + `Set("foo", f="baz")` + "\n" + + `Set("bat", f="bar")` + "\n" + + `Set("aaa", f="bbb")` + "\n" + readQueries := []string{`Row(f="bar")`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if diff := cmp.Diff(responses[0].Results, []interface{}{ &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { t.Fatal(diff) @@ -144,20 +114,68 @@ func TestExecutor_Execute_Row(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, 2) - hldr.SetBit("i", "general", 10, 3) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, 4) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("two", f=10) + Set("three", f=10) + Set("two", f=11) + Set("four", f=11)` + readQueries := []string{`Difference(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "three"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(2, f="ten") + Set(3, f="ten") + Set(2, f="eleven") + Set(4, f="eleven")` + readQueries := []string{`Difference(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("two", f="ten") + Set("three", f="ten") + Set("two", f="eleven") + Set("four", f="eleven")` + readQueries := []string{`Difference(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "three"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty difference query behaves properly. @@ -174,22 +192,73 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 1) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Intersect(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Intersect(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 200}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Intersect(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty intersect query behaves properly. @@ -204,21 +273,72 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Union(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Union(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 100, 200}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Union(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty union query behaves properly. @@ -237,44 +357,138 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { // Ensure a xor query can be executed. func TestExecutor_Execute_Xor(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Xor(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Xor(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 100}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Xor(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "f", 10, 3) - hldr.SetBit("i", "f", 10, ShardWidth+1) - hldr.SetBit("i", "f", 10, ShardWidth+2) + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+2) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + t.Fatal(err) + } else if res.Results[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res.Results[0]) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=11)` + readQueries := []string{`Count(Row(f=10))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="eleven")` + readQueries := []string{`Count(Row(f="ten"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="eleven")` + readQueries := []string{`Count(Row(f="ten"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { - t.Fatal(err) - } else if res.Results[0] != uint64(3) { - t.Fatalf("unexpected n: %d", res.Results[0]) - } } // Ensure a set query can be executed. -func TestExecutor_Execute_SetBit(t *testing.T) { - t.Run("ID", func(t *testing.T) { +func TestExecutor_Execute_Set(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -319,7 +533,25 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) }) - t.Run("Keys", func(t *testing.T) { + t.Run("RowIDColumnKey", func(t *testing.T) { + readQueries := []string{`Set("three", f=10)`} + responses := runCallTest(t, "", readQueries, + &pilosa.IndexOptions{Keys: true}) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + readQueries := []string{`Set(1, f="ten")`} + responses := runCallTest(t, "", readQueries, + nil, pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -376,6 +608,49 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) } +// Ensure a set query can be executed. +func TestExecutor_Execute_Clear(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `Set(3, f=10)` + readQueries := []string{`Clear(3, f=10)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := `Set("three", f=10)` + readQueries := []string{`Clear("three", f=10)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := `Set(1, f="ten")` + readQueries := []string{`Clear(1, f="ten")`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := `Set("one", f="ten")` + readQueries := []string{`Clear("one", f="ten")`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) +} + // Ensure a set query can be executed on a bool field. func TestExecutor_Execute_SetBool(t *testing.T) { t.Run("Basic", func(t *testing.T) { @@ -594,7 +869,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - t.Run("ID", func(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -635,7 +910,89 @@ func TestExecutor_Execute_TopN(t *testing.T) { } }) - t.Run("Keys", func(t *testing.T) { + t.Run("RowIDColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f"); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other"); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", f=0) + Set("one", f=0) + Set("sw", f=0) + Set("sw2", f=0) + Set("sw3", f=0) + Set("zero", f=10) + Set("sw", f=10) + Set("sw", f=20) + Set("zero", other=0) + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", f="zero") + Set("one", f="zero") + Set("sw", f="zero") + Set("sw2", f="zero") + Set("sw3", f="zero") + Set("zero", f="ten") + Set("sw", f="ten") + Set("sw", f="twenty") + Set("zero", other="zero") + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ + {Key: "zero", Count: 5}, + {Key: "ten", Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -820,203 +1177,443 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure Min() and Max() queries can be executed. func TestExecutor_Execute_MinMax(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { - t.Fatal(err) - } - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(0, x=0) - Set(3, x=0) - Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - Set(1, x=1) - Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) - - Set(0, f=20) - Set(1, f=-5) - Set(2, f=-5) - Set(3, f=10) - Set(` + strconv.Itoa(ShardWidth) + `, f=30) - Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) - Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) - Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) - `}); err != nil { - t.Fatal(err) - } - - t.Run("Min", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: -5, cnt: 2}, - {filter: `Row(x=0)`, exp: 10, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Min(field=f)` - } else { - pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(3, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + Set(1, x=1) + Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) + + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) }) - t.Run("Max", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 60, cnt: 1}, - {filter: `Row(x=0)`, exp: 60, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatal(err) } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Max(field=f)` - } else { - pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", x=0) + Set("three", x=0) + Set("sw1", x=0) + Set("one", x=1) + Set("sw2", x=2) + + Set("zero", f=20) + Set("one", f=-5) + Set("two", f=-5) + Set("three", f=10) + Set("sw", f=30) + Set("sw2", f=40) + Set("sw3", f=50) + Set("sw1", f=60) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) }) } // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { - t.Fatal(err) - } - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(0, x=0) - Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - - Set(0, foo=20) - Set(0, bar=2000) - Set(` + strconv.Itoa(ShardWidth) + `, foo=30) - Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) - Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) - Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) - Set(0, other=1000) - `}); err != nil { - t.Fatal(err) - } - - t.Run("NoFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + `}); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) }) - t.Run("WithFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) + if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", x=0) + Set("sw1", x=0) + + Set("zero", foo=20) + Set("zero", bar=2000) + Set("sw", foo=30) + Set("sw2", foo=40) + Set("sw3", foo=50) + Set("sw1", foo=60) + Set("zero", other=1000) + `}); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) }) } // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) - // Create index. - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - - // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { - t.Fatal(err) - } - - // Set columns. - cc := ` - Set(2, f=1, 1999-12-31T00:00) - Set(3, f=1, 2000-01-01T00:00) - Set(4, f=1, 2000-01-02T00:00) - Set(5, f=1, 2000-02-01T00:00) - Set(6, f=1, 2001-01-01T00:00) - Set(7, f=1, 2002-01-01T02:00) - - Set(2, f=1, 1999-12-30T00:00) - Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00) - ` - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { - t.Fatal(err) - } - - t.Run("Standard", func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear( 2, f=1)`, + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) }) - t.Run("Clear", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f=1)`}); err != nil { - t.Fatal(err) - } + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f=1, 1999-12-31T00:00) + Set("three", f=1, 2000-01-01T00:00) + Set("four", f=1, 2000-01-02T00:00) + Set("five", f=1, 2000-02-01T00:00) + Set("six", f=1, 2001-01-01T00:00) + Set("seven", f=1, 2002-01-01T02:00) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) + Set("two", f=1, 1999-12-30T00:00) + Set("two", f=1, 2002-02-01T00:00) + Set("two", f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear("two", f=1)`, + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) + + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear( 2, f="foo")`, + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f="foo", 1999-12-31T00:00) + Set("three", f="foo", 2000-01-01T00:00) + Set("four", f="foo", 2000-01-02T00:00) + Set("five", f="foo", 2000-02-01T00:00) + Set("six", f="foo", 2001-01-01T00:00) + Set("seven", f="foo", 2002-01-01T02:00) + + Set("two", f="foo", 1999-12-30T00:00) + Set("two", f="foo", 2002-02-01T00:00) + Set("two", f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear("two", f="foo")`, + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) }) } @@ -1426,144 +2023,98 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { } -func TestExecutor_QueryCall(t *testing.T) { +func TestExecutor_ExecuteOptions(t *testing.T) { t.Run("excludeRowAttrs", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeRowAttrs=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{`Options(Row(f=10), excludeRowAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("excludeColumns", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeColumns=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{`Options(Row(f=10), excludeColumns=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("columnAttrs", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + writeQuery := ` Set(100, f=10) - SetColumnAttrs(100, foo="bar") - `}); err != nil { - t.Fatal(err) - } - + SetColumnAttrs(100, foo="bar")` + readQueries := []string{`Options(Row(f=10), columnAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) targetColAttrSets := []*pilosa.ColumnAttrSet{ {ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), columnAttrs=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + } else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + + t.Run("columnAttrsWithKeys", func(t *testing.T) { + writeQuery := ` + Set("one-hundred", f="ten") + SetColumnAttrs("one-hundred", foo="bar")` + readQueries := []string{`Options(Row(f="ten"), columnAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + + targetColAttrSets := []*pilosa.ColumnAttrSet{ + {Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}}, + } + + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("shards", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` - Set(100, f=10) - Set(%d, f=10) - Set(%d, f=10) - `, ShardWidth, ShardWidth*2)}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), shards=[0, 2])`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { + writeQuery := fmt.Sprintf(` + Set(100, f=10) + Set(%d, f=10) + Set(%d, f=10)`, ShardWidth, ShardWidth*2) + readQueries := []string{`Options(Row(f=10), shards=[0, 2])`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { t.Fatalf("unexpected columns: %+v", bits) } }) t.Run("multipleOpt", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{ + `Options(Row(f=10), excludeColumns=true) + Options(Row(f=10), excludeRowAttrs=true)`, } - - req := &pilosa.QueryRequest{ - Index: "i", - Query: `Options(Row(f=10), excludeColumns=true)Options(Row(f=10), excludeRowAttrs=true)`, - } - if res, err := c[0].API.Query(context.Background(), req); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) - } else if bits := res.Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := responses[0].Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[0].Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) @@ -1617,170 +2168,157 @@ func TestExecutor_Execute_Existence(t *testing.T) { // Ensure a not query can be executed. func TestExecutor_Execute_Not(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20) + readQueries := []string{ + `Not(Row(f=20))`, + `Not(Row(f=0))`, + `Not(Union(Row(f=10), Row(f=20)))`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}) - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), - }); err != nil { - t.Fatal(err) - } + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } - // Populated row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=20))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { - t.Fatalf("unexpected columns: %+v", bits) - } + if bits := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", bits) + } - // Populated row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=0))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", bits) - } + if bits := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) - // All existing. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Union(Row(f=10), Row(f=20)))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected columns: %+v", bits) - } + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f=10) + Set("sw1", f=10) + Set("sw2", f=20)` + readQueries := []string{`Not(Row(f=20))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{ + TrackExistence: true, + Keys: true, + }) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := fmt.Sprintf(` + Set(3, f="ten") + Set(%d, f="ten") + Set(%d, f="twenty")`, ShardWidth+1, ShardWidth+2) + readQueries := []string{`Not(Row(f="twenty"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldKeys(), + ) + if cols := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", cols) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f="ten") + Set("sw1", f="ten") + Set("sw2", f="twenty")` + readQueries := []string{`Not(Row(f="twenty"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{ + TrackExistence: true, + Keys: true, + }, pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure a row can be cleared. func TestExecutor_Execute_ClearRow(t *testing.T) { + // Set and Mutex tests use the same data and queries + writeQuery := `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20) + + readQueries := []string{ + `Row(f=10)`, + `ClearRow(f=10)`, + `ClearRow(f=10)`, + `Row(f=10)`, + `Row(f=20)`, + } + t.Run("Set", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Clear the row again and ensure we get a `false` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); res { + if res := responses[2].Results[0].(bool); res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + if bits := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + if bits := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } }) + t.Run("Mutex", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0)) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldTypeMutex("none", 0)) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Clear the row again and ensure we get a `false` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); res { + if res := responses[2].Results[0].(bool); res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + if bits := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + if bits := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } }) - t.Run("Time", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) - if err != nil { - t.Fatal(err) - } - // Set columns. - cc := ` + t.Run("Time", func(t *testing.T) { + writeQuery := ` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) @@ -1790,42 +2328,36 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00) - ` - rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)` - rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)` - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { - t.Fatal(err) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `ClearRow(f=1)`, + `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`, } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + if columns := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { t.Fatalf("unexpected columns: %+v", columns) } }) + t.Run("Int", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -1841,6 +2373,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { t.Fatal("expected clear row to return an error") } }) + t.Run("TopN", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -1913,6 +2446,145 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { }) } +// Ensure a row can be set. +func TestExecutor_Execute_SetRow(t *testing.T) { + t.Run("Set_NewRow", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + if _, err := index.CreateField("tmp", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_NoSource", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a row that does exist. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_ExistingDestination", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into an existing row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() @@ -1950,3 +2622,42 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } + +func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse { + if indexOptions == nil { + indexOptions = &pilosa.IndexOptions{} + } + + c := test.MustRunCluster(t, 1) + defer c.Close() + + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", *indexOptions) + _, err := index.CreateField("f", fieldOption...) + if err != nil { + t.Fatal(err) + } + if writeQuery != "" { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: writeQuery, + }); err != nil { + t.Fatal(err) + } + } + + responses := []pilosa.QueryResponse{} + for _, query := range readQueries { + res, err := c[0].API.Query(context.Background(), + &pilosa.QueryRequest{ + Index: "i", + Query: query, + }) + if err != nil { + t.Fatal(err) + } + responses = append(responses, res) + } + + return responses +} diff --git a/field.go b/field.go index 27de57d8f..63bb08b63 100644 --- a/field.go +++ b/field.go @@ -15,6 +15,7 @@ package pilosa import ( + "bufio" "encoding/json" "fmt" "io/ioutil" @@ -230,11 +231,10 @@ func (f *Field) AvailableShards() *roaring.Bitmap { return b } -// addRemoteAvailableShards merges the set of available shards into the current known set +// AddRemoteAvailableShards merges the set of available shards into the current known set // and saves the set to a file. -func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error { +func (f *Field) AddRemoteAvailableShards(b *roaring.Bitmap) error { f.mergeRemoteAvailableShards(b) - // Save the updated bitmap to the data store. return f.saveAvailableShards() } @@ -246,6 +246,70 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) { f.remoteAvailableShards = f.remoteAvailableShards.Union(b) } +// loadAvailableShards reads remoteAvailableShards data for the field, if any. +func (f *Field) loadAvailableShards() error { + bm := roaring.NewBitmap() + // Read data from meta file. + path := filepath.Join(f.path, ".available.shards") + buf, err := ioutil.ReadFile(path) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "reading available shards") + } else { + if err := bm.UnmarshalBinary(buf); err != nil { + return errors.Wrap(err, "unmarshaling") + } + } + // Merge bitmap from file into field. + f.mergeRemoteAvailableShards(bm) + + return nil +} + +// saveAvailableShards writes remoteAvailableShards data for the field. +func (f *Field) saveAvailableShards() error { + f.mu.Lock() + defer f.mu.Unlock() + return f.unprotectedSaveAvailableShards() +} + +func (f *Field) unprotectedSaveAvailableShards() error { + // Open or create file. + path := filepath.Join(f.path, ".available.shards") + + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return errors.Wrap(err, "opening available shards file") + } + defer file.Close() + + // Write available shards to file. + bw := bufio.NewWriter(file) + if _, err = f.remoteAvailableShards.WriteTo(bw); err != nil { + return errors.Wrap(err, "writing bitmap to buffer") + } + bw.Flush() + + return nil +} + +// RemoveAvailableShard removes a shard from the bitmap cache. +// +// NOTE: This can be overridden on the next sync so all nodes should be updated. +func (f *Field) RemoveAvailableShard(v uint64) error { + f.mu.Lock() + defer f.mu.Unlock() + + b := f.remoteAvailableShards.Clone() + if _, err := b.Remove(v); err != nil { + return err + } + f.remoteAvailableShards = b + + return f.unprotectedSaveAvailableShards() +} + // Type returns the field type. func (f *Field) Type() string { f.mu.RLock() @@ -480,47 +544,6 @@ func (f *Field) applyOptions(opt FieldOptions) error { return nil } -// loadAvailableShards reads remoteAvailableShards data for the field, if any. -func (f *Field) loadAvailableShards() error { - bm := roaring.NewBitmap() - - // Read data from meta file. - buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards")) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "reading available shards") - } else { - if err := bm.UnmarshalBinary(buf); err != nil { - return errors.Wrap(err, "unmarshaling") - } - } - - // Merge bitmap from file into field. - f.mergeRemoteAvailableShards(bm) - - return nil -} - -// saveAvailableShards writes remoteAvailableShards data for the field. -func (f *Field) saveAvailableShards() error { - // Open or create file. - file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - return errors.Wrap(err, "opening available shards file") - } - - f.mu.RLock() - defer f.mu.RUnlock() - - // Write available shards to file. - if _, err := f.remoteAvailableShards.WriteTo(file); err != nil { - return errors.Wrap(err, "writing bitmap to buffer") - } - - return nil -} - // Close closes the field and its views. func (f *Field) Close() error { f.mu.Lock() diff --git a/field_internal_test.go b/field_internal_test.go index d72a3fd6a..c2495e9c0 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -349,7 +349,7 @@ func TestField_PersistAvailableShards(t *testing.T) { // bm represents remote available shards. bm := roaring.NewBitmap(1, 2, 3) - if err := f.addRemoteAvailableShards(bm); err != nil { + if err := f.AddRemoteAvailableShards(bm); err != nil { t.Fatal(err) } @@ -361,3 +361,44 @@ func TestField_PersistAvailableShards(t *testing.T) { } } + +// Ensure that persisting available shards having a smaller footprint (for example, +// when going from a bitmap to a smaller, RLE representation) succeeds. +func TestField_PersistAvailableShardsFootprint(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap() + for i := uint64(0); i < 1204; i += 2 { + bm.Add(i) + } + + if err := f.AddRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) { + t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice()) + } + + bm1 := roaring.NewBitmap() + for i := uint64(1); i < 1204; i += 2 { + bm1.Add(i) + } + + if err := f.AddRemoteAvailableShards(bm1); err != nil { + t.Fatal(err) + } + + // Reload field and verify that shard data is persisted. + result := bm.Union(bm1) + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), result.Slice()) { + t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice()) + } + +} diff --git a/field_test.go b/field_test.go index 520179f52..2a0efc0b9 100644 --- a/field_test.go +++ b/field_test.go @@ -18,7 +18,9 @@ import ( "io/ioutil" "testing" + "github.com/google/go-cmp/cmp" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/test" ) @@ -185,3 +187,39 @@ func TestField_NameValidation(t *testing.T) { } } } + +// Ensure can update and delete available shards. +func TestField_AvailableShards(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set values on shards 0 & 2, and verify. + if _, err := f.SetBit(0, 100, nil); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, ShardWidth*2, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { + t.Fatal(diff) + } + + // Set remote shards and verify. + f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)) + if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" { + t.Fatal(diff) + } + + // Delete shards; only local shards should remain. + f.RemoveAvailableShard(0) + f.RemoveAvailableShard(1) + f.RemoveAvailableShard(2) + f.RemoveAvailableShard(3) + f.RemoveAvailableShard(4) + if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { + t.Fatal(diff) + } +} diff --git a/fragment.go b/fragment.go index ee586afc6..63c9cce37 100644 --- a/fragment.go +++ b/fragment.go @@ -352,11 +352,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row { } // Only use a subset of the containers. - // NOTE: The start & end ranges must be divisible by + // NOTE: The start & end ranges must be divisible by container width. data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. - // We Clone() data because otherwise row will contains pointers to containers in storage. + // We Clone() data because otherwise row will contain pointers to containers in storage. // This causes unexpected results when we cache the row and try to use it later. row := &Row{ segments: []rowSegment{{ @@ -491,6 +491,56 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return changed, nil } +// setRow replaces an existing row (specified by rowID) with the given +// Row. This updates both the on-disk storage and the in-cache bitmap. +func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.unprotectedSetRow(row, rowID) +} + +func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) { + // TODO: In order to return `changed`, we need to first compare + // the existing row with the given row. Determine if the overhead + // of this is worth having `changed`. + // For now we will assume changed is always true. + changed = true + + // First container of the row in storage. + headContainerKey := rowID << shardVsContainerExponent + + // Remove every existing container in the row. + for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { + f.storage.Containers.Remove(headContainerKey + i) + } + + // From the given row, get the rowSegment for this shard. + seg := row.segment(f.shard) + if seg == nil { + return changed, nil + } + + // Put each container from rowSegment to fragment storage. + citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) + for citer.Next() { + k, c := citer.Value() + f.storage.Containers.Put(headContainerKey+(k%(1< 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.ShardID != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.ShardID)) + } + return i, nil +} + func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2508,6 +2579,23 @@ func (m *DeleteFieldMessage) Size() (n int) { return n } +func (m *DeleteAvailableShardMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.ShardID != 0 { + n += 1 + sovPrivate(uint64(m.ShardID)) + } + return n +} + func (m *Field) Size() (n int) { var l int _ = l @@ -4442,6 +4530,133 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteAvailableShardMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteAvailableShardMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ShardID", wireType) + } + m.ShardID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ShardID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7184,74 +7399,75 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1095 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44, - 0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0xdb, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x15, - 0x0d, 0x95, 0x08, 0x55, 0x7b, 0xc3, 0x5f, 0xa5, 0x92, 0x6c, 0x28, 0xa6, 0x24, 0x94, 0x71, 0x92, - 0xbb, 0x5e, 0x4c, 0x76, 0x47, 0x8d, 0x15, 0xaf, 0xc7, 0xd8, 0xe3, 0x24, 0xdb, 0x0b, 0x6e, 0x41, - 0xe2, 0x05, 0x10, 0x4f, 0xc2, 0x23, 0x70, 0xc9, 0x23, 0xa0, 0xf0, 0x22, 0x68, 0xce, 0x8c, 0x7f, - 0xb2, 0xd9, 0xb0, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f, - 0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x46, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c, - 0x26, 0x4f, 0xa1, 0x1b, 0x24, 0x23, 0x7e, 0xb6, 0xc3, 0x25, 0xf3, 0x3c, 0x70, 0x9f, 0xf1, 0x49, - 0xee, 0x3b, 0x6b, 0xd6, 0x7a, 0x87, 0xe2, 0xd9, 0xfb, 0x10, 0x16, 0xf7, 0x32, 0x36, 0x3c, 0xde, - 0x3e, 0x8b, 0x72, 0xc9, 0x93, 0x21, 0xf7, 0x5d, 0xd4, 0x4e, 0xa1, 0xe4, 0x0f, 0x0b, 0x6e, 0x7c, - 0x1d, 0xf1, 0x78, 0xf4, 0x7d, 0x2a, 0x23, 0x91, 0xe4, 0xde, 0x7b, 0xd0, 0xdd, 0x62, 0xc3, 0x23, - 0xbe, 0x37, 0x49, 0x39, 0x7a, 0xec, 0xd2, 0x1a, 0xa8, 0xb4, 0x61, 0xf4, 0x4a, 0x7b, 0xec, 0xd3, - 0x1a, 0xf0, 0xd6, 0xa0, 0xb7, 0x17, 0x8d, 0xf9, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf6, 0x5b, 0x78, - 0xbb, 0x09, 0x29, 0xaa, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0x4b, 0xe0, 0xec, 0x44, 0x89, 0xdf, - 0x5d, 0xb3, 0xd6, 0x1d, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0x27, 0xf6, - 0xea, 0x27, 0x12, 0x02, 0x8b, 0xc1, 0x38, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, - 0xed, 0x2c, 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xda, 0x8c, 0xc5, 0xf0, 0x78, 0xc0, - 0x24, 0xa3, 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x3b, 0x63, 0xa7, 0x05, 0x85, - 0x62, 0x1e, 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, - 0x47, 0x2c, 0x1b, 0x61, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, - 0x26, 0x01, 0x2c, 0x37, 0xe2, 0x1b, 0x9a, 0x2b, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe4, 0xbe, 0xb5, - 0xe6, 0xac, 0xbb, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0xe3, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, - 0xb9, 0x03, 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, - 0x19, 0xd2, 0xc8, 0xbd, 0xc7, 0xd0, 0x09, 0x25, 0x4b, 0x46, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfc, - 0x60, 0xa3, 0x6c, 0x9c, 0x8d, 0xca, 0x6c, 0xa3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa1, 0xd5, 0x95, - 0x77, 0xbf, 0x80, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0x27, 0x65, 0x56, 0x8f, 0xf9, 0x44, 0xbd, - 0xff, 0x84, 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xdb, 0x9f, 0x5a, 0xe4, 0x00, 0xbc, - 0xad, 0x8c, 0x33, 0xc9, 0x31, 0xc8, 0x0e, 0xcf, 0x73, 0xf6, 0x92, 0x5f, 0x9d, 0x71, 0x9d, 0x45, - 0xbb, 0x99, 0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x3e, 0x78, 0x03, 0x1e, 0x73, 0xc9, 0x4d, - 0xd7, 0xff, 0x87, 0x5f, 0x12, 0x96, 0x1c, 0xe6, 0xdb, 0x7a, 0xf7, 0xc0, 0x55, 0x23, 0x84, 0x14, - 0x7a, 0x0f, 0x6f, 0xd5, 0x79, 0xaa, 0xa6, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0x67, 0xee, - 0xc3, 0x66, 0xb4, 0xd2, 0x7d, 0x13, 0xca, 0xc1, 0x50, 0x2b, 0x75, 0xa8, 0xe6, 0xf8, 0x99, 0x68, - 0x4f, 0xca, 0xe7, 0x5e, 0x37, 0x1a, 0x79, 0x61, 0x50, 0xd5, 0x95, 0xbb, 0x6c, 0xcc, 0xcd, 0x1d, - 0x3c, 0x57, 0x54, 0xec, 0xf9, 0x54, 0x94, 0x7b, 0xd5, 0xc9, 0x6a, 0xbb, 0x38, 0xca, 0x3d, 0x0a, - 0xe4, 0x11, 0xb4, 0xc3, 0xe1, 0x11, 0x1f, 0x33, 0xef, 0x23, 0x58, 0x40, 0x1e, 0x3c, 0x37, 0xcd, - 0x76, 0x73, 0x2a, 0x89, 0xb4, 0xd4, 0x93, 0x81, 0xe1, 0x3f, 0x93, 0xd3, 0x3d, 0x68, 0x63, 0xf4, - 0xdc, 0x77, 0xa7, 0xdd, 0x20, 0x4e, 0x8d, 0x9a, 0x6c, 0x83, 0xb3, 0x4f, 0x03, 0x35, 0x44, 0xc8, - 0xa0, 0xf4, 0x62, 0x24, 0xe5, 0xfb, 0x1b, 0x91, 0x4b, 0x93, 0x0d, 0x3c, 0x2b, 0xec, 0xb9, 0xc8, - 0x24, 0xa6, 0xbe, 0x4f, 0xf1, 0x4c, 0x5e, 0x80, 0xbb, 0x2b, 0x46, 0xdc, 0x5b, 0x04, 0x3b, 0x18, - 0x18, 0x1f, 0x76, 0x30, 0xf0, 0xde, 0x47, 0xf7, 0x26, 0x35, 0xfd, 0x9a, 0xc4, 0x3e, 0x0d, 0x28, - 0x06, 0xbe, 0x0b, 0xfd, 0x20, 0xdf, 0x12, 0x22, 0x1b, 0x45, 0x09, 0x93, 0x22, 0x33, 0x6b, 0xf7, - 0x22, 0x48, 0x9e, 0xc0, 0x92, 0x72, 0x1f, 0x4a, 0x26, 0x79, 0x59, 0xbf, 0x15, 0x68, 0x2b, 0xac, - 0x0a, 0x67, 0x24, 0x1c, 0x04, 0x65, 0x57, 0x56, 0x10, 0x05, 0xf2, 0x9d, 0xf6, 0xb0, 0x7d, 0xc2, - 0x13, 0xd9, 0xe8, 0x00, 0x94, 0xd1, 0x41, 0x9f, 0x6a, 0xc1, 0x23, 0xfa, 0x29, 0x86, 0xf3, 0x62, - 0xcd, 0x59, 0xa1, 0x14, 0x75, 0xe4, 0x57, 0x0b, 0xa0, 0x24, 0x54, 0xe4, 0xd5, 0x15, 0xeb, 0xea, - 0x2b, 0xde, 0x7a, 0x59, 0x63, 0xd3, 0xb2, 0x4b, 0xb5, 0x95, 0xc6, 0x69, 0xd9, 0x03, 0x9f, 0xd4, - 0x3d, 0xa0, 0x8b, 0x77, 0x7b, 0xaa, 0x07, 0x74, 0xd4, 0xba, 0x13, 0x9e, 0x43, 0xaf, 0x81, 0xcf, - 0xec, 0x87, 0x8f, 0xab, 0x7e, 0xb0, 0xa7, 0x5d, 0x22, 0x6e, 0x5c, 0x96, 0x5d, 0xf1, 0x0c, 0x7a, - 0x0d, 0x78, 0xa6, 0xc7, 0x75, 0xb8, 0xf9, 0xd5, 0x09, 0x8b, 0x62, 0x76, 0x18, 0xeb, 0xf5, 0x54, - 0x2e, 0xd9, 0x69, 0x98, 0x44, 0xd0, 0xdf, 0x8a, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, - 0x03, 0x55, 0xf1, 0x6a, 0x60, 0x76, 0xfd, 0xbc, 0xbb, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, - 0xc7, 0x5a, 0x49, 0x0e, 0xa0, 0xb3, 0x19, 0x06, 0x4f, 0x33, 0x51, 0xa4, 0x33, 0x49, 0x97, 0x1f, - 0x4c, 0xfb, 0xf2, 0x07, 0xd3, 0xb9, 0xf4, 0xc1, 0x74, 0xab, 0x0f, 0x26, 0x09, 0x61, 0x59, 0xef, - 0x2b, 0x35, 0xaf, 0xd7, 0x59, 0x57, 0xe5, 0xd7, 0xcc, 0x69, 0x7c, 0xcd, 0x42, 0x58, 0xd6, 0x6b, - 0xe9, 0x4d, 0x3a, 0xfd, 0xdd, 0x86, 0x65, 0xca, 0xf3, 0xe8, 0x15, 0x0f, 0x92, 0x5c, 0x66, 0xc5, - 0x50, 0x6d, 0x1f, 0x75, 0xff, 0x5b, 0x71, 0x68, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x4e, 0xa7, 0x7b, - 0x0f, 0xa0, 0x37, 0x3d, 0x9d, 0x97, 0x4d, 0x9b, 0x26, 0xde, 0x03, 0x58, 0x08, 0x45, 0x91, 0x0d, - 0xab, 0xf6, 0x6d, 0x6c, 0x44, 0xcd, 0x4c, 0xab, 0x69, 0x69, 0xd6, 0x18, 0x8d, 0xd6, 0x9c, 0xd1, - 0x78, 0x3c, 0xd5, 0x4a, 0x7e, 0x1b, 0x2f, 0xbc, 0x53, 0x5f, 0xb8, 0xa0, 0xa6, 0x17, 0xad, 0xc9, - 0x2f, 0x16, 0xdc, 0x68, 0x52, 0x78, 0xad, 0xc1, 0xad, 0x2a, 0x62, 0xcf, 0xac, 0x88, 0x33, 0xab, - 0x22, 0x6e, 0x5d, 0x91, 0xfa, 0xc3, 0xdc, 0x6a, 0x7c, 0x98, 0xc9, 0x31, 0xdc, 0xb9, 0x54, 0xa6, - 0x2d, 0x31, 0x4e, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, - 0x05, 0xf2, 0x19, 0xdc, 0x0e, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x35, 0x70, 0x76, 0xf9, 0xe9, - 0x15, 0xcf, 0x57, 0x2a, 0xf2, 0x25, 0xf8, 0xfb, 0xe9, 0x88, 0x49, 0x7e, 0xad, 0xdb, 0x9b, 0xd0, - 0xd9, 0x13, 0xa9, 0x88, 0xc5, 0xcb, 0xc9, 0x9c, 0xa9, 0xf7, 0x61, 0x41, 0xef, 0x6f, 0xbd, 0x46, - 0xba, 0xb4, 0x14, 0xc9, 0x2d, 0xd5, 0xd0, 0x43, 0x16, 0x0f, 0x8b, 0x58, 0xd1, 0x50, 0x3f, 0x6d, - 0xf9, 0xe6, 0xd2, 0x9f, 0xe7, 0xab, 0xd6, 0x5f, 0xe7, 0xab, 0xd6, 0xdf, 0xe7, 0xab, 0xd6, 0x6f, - 0xff, 0xac, 0xbe, 0x75, 0xd8, 0xc6, 0x9f, 0xfa, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x31, - 0x07, 0xf3, 0xac, 0xe5, 0x0b, 0x00, 0x00, + // 1113 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, + 0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1, + 0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9, + 0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90, + 0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76, + 0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c, + 0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04, + 0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7, + 0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4, + 0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f, + 0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6, + 0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82, + 0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21, + 0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d, + 0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22, + 0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf, + 0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28, + 0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87, + 0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5, + 0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82, + 0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6, + 0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86, + 0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30, + 0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e, + 0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73, + 0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96, + 0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce, + 0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15, + 0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f, + 0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0, + 0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce, + 0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d, + 0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b, + 0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8, + 0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1, + 0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59, + 0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0, + 0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36, + 0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45, + 0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09, + 0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a, + 0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42, + 0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63, + 0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c, + 0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4, + 0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2, + 0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c, + 0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68, + 0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a, + 0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a, + 0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75, + 0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38, + 0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0, + 0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c, + 0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65, + 0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8, + 0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26, + 0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca, + 0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e, + 0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96, + 0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde, + 0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c, + 0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea, + 0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6, + 0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde, + 0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad, + 0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea, + 0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a, + 0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff, + 0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 88d16363f..57b98f62c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -68,6 +68,12 @@ message DeleteFieldMessage { string Field = 2; } +message DeleteAvailableShardMessage { + string Index = 1; + string Field = 2; + uint64 ShardID = 3; +} + message Field { string Name = 1; FieldOptions Meta = 2; diff --git a/pilosa.go b/pilosa.go index 7378eb628..9798ff315 100644 --- a/pilosa.go +++ b/pilosa.go @@ -117,7 +117,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { - ID uint64 `json:"id"` + ID uint64 `json:"id,omitempty"` Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } diff --git a/pql/pql.peg b/pql/pql.peg index c38046f67..9e0087046 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -11,6 +11,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} + / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 5adc244f8..2d5877fbb 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -64,9 +64,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 - rulePegText ruleAction14 ruleAction15 + rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -101,6 +101,8 @@ const ( ruleAction47 ruleAction48 ruleAction49 + ruleAction50 + ruleAction51 ) var rul3s = [...]string{ @@ -153,9 +155,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", - "PegText", "Action14", "Action15", + "PegText", "Action16", "Action17", "Action18", @@ -190,6 +192,8 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", + "Action50", + "Action51", } type token32 struct { @@ -306,7 +310,7 @@ type PQL struct { Buffer string buffer []rune - rules [86]func() bool + rules [88]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -419,84 +423,88 @@ func (p *PQL) Execute() { case ruleAction9: p.endCall() case ruleAction10: - p.startCall("TopN") + p.startCall("Store") case ruleAction11: p.endCall() case ruleAction12: - p.startCall("Range") + p.startCall("TopN") case ruleAction13: p.endCall() case ruleAction14: - p.startCall(buffer[begin:end]) + p.startCall("Range") case ruleAction15: p.endCall() case ruleAction16: - p.addBTWN() + p.startCall(buffer[begin:end]) case ruleAction17: - p.addLTE() + p.endCall() case ruleAction18: - p.addGTE() + p.addBTWN() case ruleAction19: - p.addEQ() + p.addLTE() case ruleAction20: - p.addNEQ() + p.addGTE() case ruleAction21: - p.addLT() + p.addEQ() case ruleAction22: - p.addGT() + p.addNEQ() case ruleAction23: - p.startConditional() + p.addLT() case ruleAction24: - p.endConditional() + p.addGT() case ruleAction25: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction26: - p.condAdd(buffer[begin:end]) + p.endConditional() case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.addPosStr("_start", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction29: - p.addPosStr("_end", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction30: - p.startList() + p.addPosStr("_start", buffer[begin:end]) case ruleAction31: - p.endList() + p.addPosStr("_end", buffer[begin:end]) case ruleAction32: - p.addVal(nil) + p.startList() case ruleAction33: - p.addVal(true) + p.endList() case ruleAction34: - p.addVal(false) + p.addVal(nil) case ruleAction35: - p.addNumVal(buffer[begin:end]) + p.addVal(true) case ruleAction36: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction37: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction38: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction39: p.addVal(buffer[begin:end]) case ruleAction40: - p.addField(buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction41: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction42: - p.addPosNum("_row", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction43: - p.addPosNum("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction44: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction45: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction46: p.addPosNum("_row", buffer[begin:end]) + case ruleAction45: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction46: + p.addPosStr("_col", buffer[begin:end]) case ruleAction47: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction49: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -609,7 +617,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('T' 'o' 'p' 'N' Action10 open posfield (comma allargs)? close Action11) / ('R' 'a' 'n' 'g' 'e' Action12 open (timerange / conditional / arg) close Action13) / ( Action14 open allargs comma? close Action15))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'a' 'n' 'g' 'e' Action14 open (timerange / conditional / arg) close Action15) / ( Action16 open allargs comma? close Action17))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -658,7 +666,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction49, position) + add(ruleAction51, position) } add(ruletimestamp, position12) } @@ -744,7 +752,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction46, position) + add(ruleAction48, position) } goto l19 l20: @@ -765,7 +773,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction47, position) + add(ruleAction49, position) } goto l19 l23: @@ -786,7 +794,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction50, position) } } l19: @@ -981,7 +989,11 @@ func (p *PQL) Init() { goto l7 l35: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('T') { + if buffer[position] != rune('S') { + goto l38 + } + position++ + if buffer[position] != rune('t') { goto l38 } position++ @@ -989,11 +1001,11 @@ func (p *PQL) Init() { goto l38 } position++ - if buffer[position] != rune('p') { + if buffer[position] != rune('r') { goto l38 } position++ - if buffer[position] != rune('N') { + if buffer[position] != rune('e') { goto l38 } position++ @@ -1003,22 +1015,15 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l38 } - if !_rules[ruleposfield]() { + if !_rules[ruleCall]() { goto l38 } - { - position40, tokenIndex40 := position, tokenIndex - if !_rules[rulecomma]() { - goto l40 - } - if !_rules[ruleallargs]() { - goto l40 - } - goto l41 - l40: - position, tokenIndex = position40, tokenIndex40 + if !_rules[rulecomma]() { + goto l38 + } + if !_rules[rulearg]() { + goto l38 } - l41: if !_rules[ruleclose]() { goto l38 } @@ -1028,193 +1033,240 @@ func (p *PQL) Init() { goto l7 l38: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l43 + if buffer[position] != rune('T') { + goto l41 } position++ - if buffer[position] != rune('a') { - goto l43 + if buffer[position] != rune('o') { + goto l41 } position++ - if buffer[position] != rune('n') { - goto l43 + if buffer[position] != rune('p') { + goto l41 } position++ - if buffer[position] != rune('g') { - goto l43 - } - position++ - if buffer[position] != rune('e') { - goto l43 + if buffer[position] != rune('N') { + goto l41 } position++ { add(ruleAction12, position) } if !_rules[ruleopen]() { - goto l43 + goto l41 + } + if !_rules[ruleposfield]() { + goto l41 } { - position45, tokenIndex45 := position, tokenIndex - { - position47 := position - if !_rules[rulefield]() { - goto l46 - } - if !_rules[rulesp]() { - goto l46 - } - if buffer[position] != rune('=') { - goto l46 - } - position++ - if !_rules[rulesp]() { - goto l46 - } - if !_rules[rulevalue]() { - goto l46 - } - if !_rules[rulecomma]() { - goto l46 - } - { - position48 := position - if !_rules[ruletimestampfmt]() { - goto l46 - } - add(rulePegText, position48) - } - { - add(ruleAction28, position) - } - if !_rules[rulecomma]() { - goto l46 - } - { - position50 := position - if !_rules[ruletimestampfmt]() { - goto l46 - } - add(rulePegText, position50) - } - { - add(ruleAction29, position) - } - add(ruletimerange, position47) - } - goto l45 - l46: - position, tokenIndex = position45, tokenIndex45 - { - position53 := position - { - add(ruleAction23, position) - } - if !_rules[rulecondint]() { - goto l52 - } - if !_rules[rulecondLT]() { - goto l52 - } - { - position55 := position - { - position56 := position - if !_rules[rulefieldExpr]() { - goto l52 - } - add(rulePegText, position56) - } - if !_rules[rulesp]() { - goto l52 - } - { - add(ruleAction27, position) - } - add(rulecondfield, position55) - } - if !_rules[rulecondLT]() { - goto l52 - } - if !_rules[rulecondint]() { - goto l52 - } - { - add(ruleAction24, position) - } - add(ruleconditional, position53) - } - goto l45 - l52: - position, tokenIndex = position45, tokenIndex45 - if !_rules[rulearg]() { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulecomma]() { goto l43 } + if !_rules[ruleallargs]() { + goto l43 + } + goto l44 + l43: + position, tokenIndex = position43, tokenIndex43 } - l45: + l44: if !_rules[ruleclose]() { - goto l43 + goto l41 } { add(ruleAction13, position) } goto l7 - l43: + l41: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l46 + } + position++ + if buffer[position] != rune('a') { + goto l46 + } + position++ + if buffer[position] != rune('n') { + goto l46 + } + position++ + if buffer[position] != rune('g') { + goto l46 + } + position++ + if buffer[position] != rune('e') { + goto l46 + } + position++ + { + add(ruleAction14, position) + } + if !_rules[ruleopen]() { + goto l46 + } + { + position48, tokenIndex48 := position, tokenIndex + { + position50 := position + if !_rules[rulefield]() { + goto l49 + } + if !_rules[rulesp]() { + goto l49 + } + if buffer[position] != rune('=') { + goto l49 + } + position++ + if !_rules[rulesp]() { + goto l49 + } + if !_rules[rulevalue]() { + goto l49 + } + if !_rules[rulecomma]() { + goto l49 + } + { + position51 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position51) + } + { + add(ruleAction30, position) + } + if !_rules[rulecomma]() { + goto l49 + } + { + position53 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position53) + } + { + add(ruleAction31, position) + } + add(ruletimerange, position50) + } + goto l48 + l49: + position, tokenIndex = position48, tokenIndex48 + { + position56 := position + { + add(ruleAction25, position) + } + if !_rules[rulecondint]() { + goto l55 + } + if !_rules[rulecondLT]() { + goto l55 + } + { + position58 := position + { + position59 := position + if !_rules[rulefieldExpr]() { + goto l55 + } + add(rulePegText, position59) + } + if !_rules[rulesp]() { + goto l55 + } + { + add(ruleAction29, position) + } + add(rulecondfield, position58) + } + if !_rules[rulecondLT]() { + goto l55 + } + if !_rules[rulecondint]() { + goto l55 + } + { + add(ruleAction26, position) + } + add(ruleconditional, position56) + } + goto l48 + l55: + position, tokenIndex = position48, tokenIndex48 + if !_rules[rulearg]() { + goto l46 + } + } + l48: + if !_rules[ruleclose]() { + goto l46 + } + { + add(ruleAction15, position) + } + goto l7 + l46: position, tokenIndex = position7, tokenIndex7 { - position60 := position + position63 := position { - position61 := position + position64 := position { - position62, tokenIndex62 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l63 + goto l66 } position++ - goto l62 - l63: - position, tokenIndex = position62, tokenIndex62 + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l62: - l64: + l65: + l67: { - position65, tokenIndex65 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex { - position66, tokenIndex66 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l67 + goto l70 } position++ - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 + goto l69 + l70: + position, tokenIndex = position69, tokenIndex69 if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l71 + } + position++ + goto l69 + l71: + position, tokenIndex = position69, tokenIndex69 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l68 } position++ - goto l66 - l68: - position, tokenIndex = position66, tokenIndex66 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l65 - } - position++ } - l66: - goto l64 - l65: - position, tokenIndex = position65, tokenIndex65 + l69: + goto l67 + l68: + position, tokenIndex = position68, tokenIndex68 } - add(ruleIDENT, position61) + add(ruleIDENT, position64) } - add(rulePegText, position60) + add(rulePegText, position63) } { - add(ruleAction14, position) + add(ruleAction16, position) } if !_rules[ruleopen]() { goto l5 @@ -1223,20 +1275,20 @@ func (p *PQL) Init() { goto l5 } { - position70, tokenIndex70 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if !_rules[rulecomma]() { - goto l70 + goto l73 } - goto l71 - l70: - position, tokenIndex = position70, tokenIndex70 + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 } - l71: + l74: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction15, position) + add(ruleAction17, position) } } l7: @@ -1249,1487 +1301,1487 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position73, tokenIndex73 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position74 := position + position77 := position { - position75, tokenIndex75 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex if !_rules[ruleCall]() { - goto l76 - } - l77: - { - position78, tokenIndex78 := position, tokenIndex - if !_rules[rulecomma]() { - goto l78 - } - if !_rules[ruleCall]() { - goto l78 - } - goto l77 - l78: - position, tokenIndex = position78, tokenIndex78 - } - { - position79, tokenIndex79 := position, tokenIndex - if !_rules[rulecomma]() { - goto l79 - } - if !_rules[ruleargs]() { - goto l79 - } - goto l80 - l79: - position, tokenIndex = position79, tokenIndex79 + goto l79 } l80: - goto l75 - l76: - position, tokenIndex = position75, tokenIndex75 - if !_rules[ruleargs]() { - goto l81 + { + position81, tokenIndex81 := position, tokenIndex + if !_rules[rulecomma]() { + goto l81 + } + if !_rules[ruleCall]() { + goto l81 + } + goto l80 + l81: + position, tokenIndex = position81, tokenIndex81 } - goto l75 - l81: - position, tokenIndex = position75, tokenIndex75 + { + position82, tokenIndex82 := position, tokenIndex + if !_rules[rulecomma]() { + goto l82 + } + if !_rules[ruleargs]() { + goto l82 + } + goto l83 + l82: + position, tokenIndex = position82, tokenIndex82 + } + l83: + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 + if !_rules[ruleargs]() { + goto l84 + } + goto l78 + l84: + position, tokenIndex = position78, tokenIndex78 if !_rules[rulesp]() { - goto l73 + goto l76 } } - l75: - add(ruleallargs, position74) + l78: + add(ruleallargs, position77) } return true - l73: - position, tokenIndex = position73, tokenIndex73 + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position82, tokenIndex82 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex { - position83 := position + position86 := position if !_rules[rulearg]() { - goto l82 + goto l85 } { - position84, tokenIndex84 := position, tokenIndex + position87, tokenIndex87 := position, tokenIndex if !_rules[rulecomma]() { - goto l84 + goto l87 } if !_rules[ruleargs]() { - goto l84 + goto l87 } - goto l85 - l84: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l87: + position, tokenIndex = position87, tokenIndex87 } - l85: + l88: if !_rules[rulesp]() { - goto l82 + goto l85 } - add(ruleargs, position83) + add(ruleargs, position86) } return true - l82: - position, tokenIndex = position82, tokenIndex82 + l85: + position, tokenIndex = position85, tokenIndex85 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position86, tokenIndex86 := position, tokenIndex + position89, tokenIndex89 := position, tokenIndex { - position87 := position + position90 := position { - position88, tokenIndex88 := position, tokenIndex + position91, tokenIndex91 := position, tokenIndex if !_rules[rulefield]() { - goto l89 + goto l92 } if !_rules[rulesp]() { - goto l89 + goto l92 } if buffer[position] != rune('=') { - goto l89 + goto l92 } position++ if !_rules[rulesp]() { - goto l89 + goto l92 } if !_rules[rulevalue]() { + goto l92 + } + goto l91 + l92: + position, tokenIndex = position91, tokenIndex91 + if !_rules[rulefield]() { goto l89 } - goto l88 - l89: - position, tokenIndex = position88, tokenIndex88 - if !_rules[rulefield]() { - goto l86 - } if !_rules[rulesp]() { - goto l86 + goto l89 } { - position90 := position + position93 := position { - position91, tokenIndex91 := position, tokenIndex + position94, tokenIndex94 := position, tokenIndex if buffer[position] != rune('>') { - goto l92 + goto l95 } position++ if buffer[position] != rune('<') { - goto l92 - } - position++ - { - add(ruleAction16, position) - } - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('<') { - goto l94 - } - position++ - if buffer[position] != rune('=') { - goto l94 - } - position++ - { - add(ruleAction17, position) - } - goto l91 - l94: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('>') { - goto l96 - } - position++ - if buffer[position] != rune('=') { - goto l96 + goto l95 } position++ { add(ruleAction18, position) } - goto l91 - l96: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('=') { - goto l98 + goto l94 + l95: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('<') { + goto l97 } position++ if buffer[position] != rune('=') { - goto l98 + goto l97 } position++ { add(ruleAction19, position) } - goto l91 - l98: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('!') { - goto l100 + goto l94 + l97: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('>') { + goto l99 } position++ if buffer[position] != rune('=') { - goto l100 + goto l99 } position++ { add(ruleAction20, position) } - goto l91 - l100: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('<') { - goto l102 + goto l94 + l99: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('=') { + goto l101 + } + position++ + if buffer[position] != rune('=') { + goto l101 } position++ { add(ruleAction21, position) } - goto l91 - l102: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('>') { - goto l86 + goto l94 + l101: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('!') { + goto l103 + } + position++ + if buffer[position] != rune('=') { + goto l103 } position++ { add(ruleAction22, position) } - } - l91: - add(ruleCOND, position90) - } - if !_rules[rulesp]() { - goto l86 - } - if !_rules[rulevalue]() { - goto l86 - } - } - l88: - add(rulearg, position87) - } - return true - l86: - position, tokenIndex = position86, tokenIndex86 - return false - }, - /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ - nil, - /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ - nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ - func() bool { - position107, tokenIndex107 := position, tokenIndex - { - position108 := position - { - position109 := position - { - position110, tokenIndex110 := position, tokenIndex - { - position112, tokenIndex112 := position, tokenIndex - if buffer[position] != rune('-') { - goto l112 + goto l94 + l103: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('<') { + goto l105 } position++ - goto l113 - l112: - position, tokenIndex = position112, tokenIndex112 + { + add(ruleAction23, position) + } + goto l94 + l105: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('>') { + goto l89 + } + position++ + { + add(ruleAction24, position) + } } - l113: - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l111 - } - position++ - l114: + l94: + add(ruleCOND, position93) + } + if !_rules[rulesp]() { + goto l89 + } + if !_rules[rulevalue]() { + goto l89 + } + } + l91: + add(rulearg, position90) + } + return true + l89: + position, tokenIndex = position89, tokenIndex89 + return false + }, + /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ + nil, + /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ + nil, + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ + func() bool { + position110, tokenIndex110 := position, tokenIndex + { + position111 := position + { + position112 := position + { + position113, tokenIndex113 := position, tokenIndex { position115, tokenIndex115 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { + if buffer[position] != rune('-') { goto l115 } position++ - goto l114 + goto l116 l115: position, tokenIndex = position115, tokenIndex115 } - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 + l116: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l114 + } + position++ + l117: + { + position118, tokenIndex118 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l118 + } + position++ + goto l117 + l118: + position, tokenIndex = position118, tokenIndex118 + } + goto l113 + l114: + position, tokenIndex = position113, tokenIndex113 if buffer[position] != rune('0') { - goto l107 + goto l110 } position++ } - l110: - add(rulePegText, position109) + l113: + add(rulePegText, position112) } if !_rules[rulesp]() { - goto l107 + goto l110 } { - add(ruleAction25, position) + add(ruleAction27, position) } - add(rulecondint, position108) + add(rulecondint, position111) } return true - l107: - position, tokenIndex = position107, tokenIndex107 + l110: + position, tokenIndex = position110, tokenIndex110 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position117, tokenIndex117 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex { - position118 := position + position121 := position { - position119 := position + position122 := position { - position120, tokenIndex120 := position, tokenIndex + position123, tokenIndex123 := position, tokenIndex if buffer[position] != rune('<') { - goto l121 + goto l124 } position++ if buffer[position] != rune('=') { - goto l121 + goto l124 } position++ - goto l120 - l121: - position, tokenIndex = position120, tokenIndex120 + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 if buffer[position] != rune('<') { - goto l117 + goto l120 } position++ } - l120: - add(rulePegText, position119) + l123: + add(rulePegText, position122) } if !_rules[rulesp]() { - goto l117 + goto l120 } { - add(ruleAction26, position) + add(ruleAction28, position) } - add(rulecondLT, position118) + add(rulecondLT, position121) } return true - l117: - position, tokenIndex = position117, tokenIndex117 + l120: + position, tokenIndex = position120, tokenIndex120 return false }, - /* 9 condfield <- <( sp Action27)> */ + /* 9 condfield <- <( sp Action29)> */ nil, - /* 10 timerange <- <(field sp '=' sp value comma Action28 comma Action29)> */ + /* 10 timerange <- <(field sp '=' sp value comma Action30 comma Action31)> */ nil, - /* 11 value <- <(item / (lbrack Action30 list rbrack Action31))> */ + /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ func() bool { - position125, tokenIndex125 := position, tokenIndex + position128, tokenIndex128 := position, tokenIndex { - position126 := position + position129 := position { - position127, tokenIndex127 := position, tokenIndex + position130, tokenIndex130 := position, tokenIndex if !_rules[ruleitem]() { - goto l128 + goto l131 } - goto l127 - l128: - position, tokenIndex = position127, tokenIndex127 + goto l130 + l131: + position, tokenIndex = position130, tokenIndex130 { - position129 := position + position132 := position if buffer[position] != rune('[') { - goto l125 + goto l128 } position++ if !_rules[rulesp]() { - goto l125 + goto l128 } - add(rulelbrack, position129) - } - { - add(ruleAction30, position) - } - if !_rules[rulelist]() { - goto l125 - } - { - position131 := position - if !_rules[rulesp]() { - goto l125 - } - if buffer[position] != rune(']') { - goto l125 - } - position++ - if !_rules[rulesp]() { - goto l125 - } - add(rulerbrack, position131) - } - { - add(ruleAction31, position) - } - } - l127: - add(rulevalue, position126) - } - return true - l125: - position, tokenIndex = position125, tokenIndex125 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position133, tokenIndex133 := position, tokenIndex - { - position134 := position - if !_rules[ruleitem]() { - goto l133 - } - { - position135, tokenIndex135 := position, tokenIndex - if !_rules[rulecomma]() { - goto l135 - } - if !_rules[rulelist]() { - goto l135 - } - goto l136 - l135: - position, tokenIndex = position135, tokenIndex135 - } - l136: - add(rulelist, position134) - } - return true - l133: - position, tokenIndex = position133, tokenIndex133 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action35) / (<('-'? '.' [0-9]+)> Action36) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action37) / ('"' '"' Action38) / ('\'' '\'' Action39))> */ - func() bool { - position137, tokenIndex137 := position, tokenIndex - { - position138 := position - { - position139, tokenIndex139 := position, tokenIndex - if buffer[position] != rune('n') { - goto l140 - } - position++ - if buffer[position] != rune('u') { - goto l140 - } - position++ - if buffer[position] != rune('l') { - goto l140 - } - position++ - if buffer[position] != rune('l') { - goto l140 - } - position++ - { - position141, tokenIndex141 := position, tokenIndex - { - position142, tokenIndex142 := position, tokenIndex - if !_rules[rulecomma]() { - goto l143 - } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 - if !_rules[rulesp]() { - goto l140 - } - if !_rules[ruleclose]() { - goto l140 - } - } - l142: - position, tokenIndex = position141, tokenIndex141 + add(rulelbrack, position132) } { add(ruleAction32, position) } - goto l139 - l140: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('t') { - goto l145 + if !_rules[rulelist]() { + goto l128 } - position++ - if buffer[position] != rune('r') { - goto l145 - } - position++ - if buffer[position] != rune('u') { - goto l145 - } - position++ - if buffer[position] != rune('e') { - goto l145 - } - position++ { - position146, tokenIndex146 := position, tokenIndex - { - position147, tokenIndex147 := position, tokenIndex - if !_rules[rulecomma]() { - goto l148 - } - goto l147 - l148: - position, tokenIndex = position147, tokenIndex147 - if !_rules[rulesp]() { - goto l145 - } - if !_rules[ruleclose]() { - goto l145 - } + position134 := position + if !_rules[rulesp]() { + goto l128 } - l147: - position, tokenIndex = position146, tokenIndex146 + if buffer[position] != rune(']') { + goto l128 + } + position++ + if !_rules[rulesp]() { + goto l128 + } + add(rulerbrack, position134) } { add(ruleAction33, position) } + } + l130: + add(rulevalue, position129) + } + return true + l128: + position, tokenIndex = position128, tokenIndex128 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position136, tokenIndex136 := position, tokenIndex + { + position137 := position + if !_rules[ruleitem]() { + goto l136 + } + { + position138, tokenIndex138 := position, tokenIndex + if !_rules[rulecomma]() { + goto l138 + } + if !_rules[rulelist]() { + goto l138 + } goto l139 - l145: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('f') { - goto l150 + l138: + position, tokenIndex = position138, tokenIndex138 + } + l139: + add(rulelist, position137) + } + return true + l136: + position, tokenIndex = position136, tokenIndex136 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action39) / ('"' '"' Action40) / ('\'' '\'' Action41))> */ + func() bool { + position140, tokenIndex140 := position, tokenIndex + { + position141 := position + { + position142, tokenIndex142 := position, tokenIndex + if buffer[position] != rune('n') { + goto l143 } position++ - if buffer[position] != rune('a') { - goto l150 + if buffer[position] != rune('u') { + goto l143 } position++ if buffer[position] != rune('l') { - goto l150 + goto l143 } position++ - if buffer[position] != rune('s') { - goto l150 - } - position++ - if buffer[position] != rune('e') { - goto l150 + if buffer[position] != rune('l') { + goto l143 } position++ { - position151, tokenIndex151 := position, tokenIndex + position144, tokenIndex144 := position, tokenIndex { - position152, tokenIndex152 := position, tokenIndex + position145, tokenIndex145 := position, tokenIndex if !_rules[rulecomma]() { - goto l153 + goto l146 } - goto l152 - l153: - position, tokenIndex = position152, tokenIndex152 + goto l145 + l146: + position, tokenIndex = position145, tokenIndex145 if !_rules[rulesp]() { - goto l150 + goto l143 } if !_rules[ruleclose]() { - goto l150 + goto l143 } } - l152: - position, tokenIndex = position151, tokenIndex151 + l145: + position, tokenIndex = position144, tokenIndex144 } { add(ruleAction34, position) } - goto l139 - l150: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('t') { + goto l148 + } + position++ + if buffer[position] != rune('r') { + goto l148 + } + position++ + if buffer[position] != rune('u') { + goto l148 + } + position++ + if buffer[position] != rune('e') { + goto l148 + } + position++ { - position156 := position + position149, tokenIndex149 := position, tokenIndex { - position157, tokenIndex157 := position, tokenIndex - if buffer[position] != rune('-') { - goto l157 + position150, tokenIndex150 := position, tokenIndex + if !_rules[rulecomma]() { + goto l151 } - position++ - goto l158 - l157: - position, tokenIndex = position157, tokenIndex157 - } - l158: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l155 - } - position++ - l159: - { - position160, tokenIndex160 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l160 + goto l150 + l151: + position, tokenIndex = position150, tokenIndex150 + if !_rules[rulesp]() { + goto l148 } - position++ - goto l159 - l160: - position, tokenIndex = position160, tokenIndex160 - } - { - position161, tokenIndex161 := position, tokenIndex - if buffer[position] != rune('.') { - goto l161 + if !_rules[ruleclose]() { + goto l148 } - position++ - l163: - { - position164, tokenIndex164 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l164 - } - position++ - goto l163 - l164: - position, tokenIndex = position164, tokenIndex164 - } - goto l162 - l161: - position, tokenIndex = position161, tokenIndex161 } - l162: - add(rulePegText, position156) + l150: + position, tokenIndex = position149, tokenIndex149 } { add(ruleAction35, position) } - goto l139 - l155: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l148: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('f') { + goto l153 + } + position++ + if buffer[position] != rune('a') { + goto l153 + } + position++ + if buffer[position] != rune('l') { + goto l153 + } + position++ + if buffer[position] != rune('s') { + goto l153 + } + position++ + if buffer[position] != rune('e') { + goto l153 + } + position++ { - position167 := position + position154, tokenIndex154 := position, tokenIndex { - position168, tokenIndex168 := position, tokenIndex - if buffer[position] != rune('-') { - goto l168 + position155, tokenIndex155 := position, tokenIndex + if !_rules[rulecomma]() { + goto l156 } - position++ - goto l169 - l168: - position, tokenIndex = position168, tokenIndex168 - } - l169: - if buffer[position] != rune('.') { - goto l166 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l166 - } - position++ - l170: - { - position171, tokenIndex171 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l171 + goto l155 + l156: + position, tokenIndex = position155, tokenIndex155 + if !_rules[rulesp]() { + goto l153 + } + if !_rules[ruleclose]() { + goto l153 } - position++ - goto l170 - l171: - position, tokenIndex = position171, tokenIndex171 } - add(rulePegText, position167) + l155: + position, tokenIndex = position154, tokenIndex154 } { add(ruleAction36, position) } - goto l139 - l166: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l153: + position, tokenIndex = position142, tokenIndex142 { - position174 := position + position159 := position { - position177, tokenIndex177 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l178 - } - position++ - goto l177 - l178: - position, tokenIndex = position177, tokenIndex177 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l179 - } - position++ - goto l177 - l179: - position, tokenIndex = position177, tokenIndex177 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l180 - } - position++ - goto l177 - l180: - position, tokenIndex = position177, tokenIndex177 + position160, tokenIndex160 := position, tokenIndex if buffer[position] != rune('-') { - goto l181 - } - position++ - goto l177 - l181: - position, tokenIndex = position177, tokenIndex177 - if buffer[position] != rune('_') { - goto l182 - } - position++ - goto l177 - l182: - position, tokenIndex = position177, tokenIndex177 - if buffer[position] != rune(':') { - goto l173 + goto l160 } position++ + goto l161 + l160: + position, tokenIndex = position160, tokenIndex160 } - l177: - l175: + l161: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l158 + } + position++ + l162: { - position176, tokenIndex176 := position, tokenIndex - { - position183, tokenIndex183 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l184 - } - position++ - goto l183 - l184: - position, tokenIndex = position183, tokenIndex183 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l185 - } - position++ - goto l183 - l185: - position, tokenIndex = position183, tokenIndex183 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l186 - } - position++ - goto l183 - l186: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune('-') { - goto l187 - } - position++ - goto l183 - l187: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune('_') { - goto l188 - } - position++ - goto l183 - l188: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune(':') { - goto l176 - } - position++ + position163, tokenIndex163 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 } - l183: - goto l175 - l176: - position, tokenIndex = position176, tokenIndex176 + position++ + goto l162 + l163: + position, tokenIndex = position163, tokenIndex163 } - add(rulePegText, position174) + { + position164, tokenIndex164 := position, tokenIndex + if buffer[position] != rune('.') { + goto l164 + } + position++ + l166: + { + position167, tokenIndex167 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l167 + } + position++ + goto l166 + l167: + position, tokenIndex = position167, tokenIndex167 + } + goto l165 + l164: + position, tokenIndex = position164, tokenIndex164 + } + l165: + add(rulePegText, position159) } { add(ruleAction37, position) } - goto l139 - l173: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('"') { - goto l190 - } - position++ + goto l142 + l158: + position, tokenIndex = position142, tokenIndex142 { - position191 := position - if !_rules[ruledoublequotedstring]() { - goto l190 + position170 := position + { + position171, tokenIndex171 := position, tokenIndex + if buffer[position] != rune('-') { + goto l171 + } + position++ + goto l172 + l171: + position, tokenIndex = position171, tokenIndex171 } - add(rulePegText, position191) + l172: + if buffer[position] != rune('.') { + goto l169 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l169 + } + position++ + l173: + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l174 + } + position++ + goto l173 + l174: + position, tokenIndex = position174, tokenIndex174 + } + add(rulePegText, position170) } - if buffer[position] != rune('"') { - goto l190 - } - position++ { add(ruleAction38, position) } - goto l139 - l190: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('\'') { - goto l137 - } - position++ + goto l142 + l169: + position, tokenIndex = position142, tokenIndex142 { - position193 := position - if !_rules[rulesinglequotedstring]() { - goto l137 + position177 := position + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l176 + } + position++ } - add(rulePegText, position193) + l180: + l178: + { + position179, tokenIndex179 := position, tokenIndex + { + position186, tokenIndex186 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l187 + } + position++ + goto l186 + l187: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l188 + } + position++ + goto l186 + l188: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l189 + } + position++ + goto l186 + l189: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('-') { + goto l190 + } + position++ + goto l186 + l190: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('_') { + goto l191 + } + position++ + goto l186 + l191: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune(':') { + goto l179 + } + position++ + } + l186: + goto l178 + l179: + position, tokenIndex = position179, tokenIndex179 + } + add(rulePegText, position177) } - if buffer[position] != rune('\'') { - goto l137 - } - position++ { add(ruleAction39, position) } + goto l142 + l176: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + position194 := position + if !_rules[ruledoublequotedstring]() { + goto l193 + } + add(rulePegText, position194) + } + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + add(ruleAction40, position) + } + goto l142 + l193: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + position196 := position + if !_rules[rulesinglequotedstring]() { + goto l140 + } + add(rulePegText, position196) + } + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + add(ruleAction41, position) + } } - l139: - add(ruleitem, position138) + l142: + add(ruleitem, position141) } return true - l137: - position, tokenIndex = position137, tokenIndex137 + l140: + position, tokenIndex = position140, tokenIndex140 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position196 := position - l197: + position199 := position + l200: { - position198, tokenIndex198 := position, tokenIndex + position201, tokenIndex201 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex { - position201, tokenIndex201 := position, tokenIndex + position204, tokenIndex204 := position, tokenIndex { - position202, tokenIndex202 := position, tokenIndex + position205, tokenIndex205 := position, tokenIndex if buffer[position] != rune('"') { - goto l203 + goto l206 } position++ - goto l202 - l203: - position, tokenIndex = position202, tokenIndex202 + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 if buffer[position] != rune('\\') { + goto l207 + } + position++ + goto l205 + l207: + position, tokenIndex = position205, tokenIndex205 + if buffer[position] != rune('\n') { goto l204 } position++ - goto l202 - l204: - position, tokenIndex = position202, tokenIndex202 - if buffer[position] != rune('\n') { - goto l201 - } - position++ } - l202: - goto l200 - l201: - position, tokenIndex = position201, tokenIndex201 + l205: + goto l203 + l204: + position, tokenIndex = position204, tokenIndex204 } if !matchDot() { - goto l200 + goto l203 } - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l203: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l205 + goto l208 } position++ if buffer[position] != rune('n') { - goto l205 + goto l208 } position++ - goto l199 - l205: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l208: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l206 + goto l209 } position++ if buffer[position] != rune('"') { - goto l206 + goto l209 } position++ - goto l199 - l206: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l209: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l207 + goto l210 } position++ if buffer[position] != rune('\'') { - goto l207 + goto l210 } position++ - goto l199 - l207: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l210: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l198 + goto l201 } position++ if buffer[position] != rune('\\') { - goto l198 + goto l201 } position++ } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l202: + goto l200 + l201: + position, tokenIndex = position201, tokenIndex201 } - add(ruledoublequotedstring, position196) + add(ruledoublequotedstring, position199) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position209 := position - l210: + position212 := position + l213: { - position211, tokenIndex211 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position212, tokenIndex212 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex { - position214, tokenIndex214 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex { - position215, tokenIndex215 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex if buffer[position] != rune('\'') { - goto l216 + goto l219 } position++ - goto l215 - l216: - position, tokenIndex = position215, tokenIndex215 + goto l218 + l219: + position, tokenIndex = position218, tokenIndex218 if buffer[position] != rune('\\') { + goto l220 + } + position++ + goto l218 + l220: + position, tokenIndex = position218, tokenIndex218 + if buffer[position] != rune('\n') { goto l217 } position++ - goto l215 - l217: - position, tokenIndex = position215, tokenIndex215 - if buffer[position] != rune('\n') { - goto l214 - } - position++ } - l215: - goto l213 - l214: - position, tokenIndex = position214, tokenIndex214 + l218: + goto l216 + l217: + position, tokenIndex = position217, tokenIndex217 } if !matchDot() { - goto l213 + goto l216 } - goto l212 - l213: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l218 + goto l221 } position++ if buffer[position] != rune('n') { - goto l218 + goto l221 } position++ - goto l212 - l218: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l221: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l219 + goto l222 } position++ if buffer[position] != rune('"') { - goto l219 + goto l222 } position++ - goto l212 - l219: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l222: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l220 + goto l223 } position++ if buffer[position] != rune('\'') { - goto l220 + goto l223 } position++ - goto l212 - l220: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l223: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l211 + goto l214 } position++ if buffer[position] != rune('\\') { - goto l211 + goto l214 } position++ } - l212: - goto l210 - l211: - position, tokenIndex = position211, tokenIndex211 + l215: + goto l213 + l214: + position, tokenIndex = position214, tokenIndex214 } - add(rulesinglequotedstring, position209) + add(rulesinglequotedstring, position212) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position221, tokenIndex221 := position, tokenIndex + position224, tokenIndex224 := position, tokenIndex { - position222 := position + position225 := position { - position223, tokenIndex223 := position, tokenIndex + position226, tokenIndex226 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l227 + } + position++ + goto l226 + l227: + position, tokenIndex = position226, tokenIndex226 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l224 } position++ - goto l223 - l224: - position, tokenIndex = position223, tokenIndex223 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l221 - } - position++ } - l223: - l225: + l226: + l228: { - position226, tokenIndex226 := position, tokenIndex + position229, tokenIndex229 := position, tokenIndex { - position227, tokenIndex227 := position, tokenIndex + position230, tokenIndex230 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l228 - } - position++ - goto l227 - l228: - position, tokenIndex = position227, tokenIndex227 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l229 - } - position++ - goto l227 - l229: - position, tokenIndex = position227, tokenIndex227 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l230 - } - position++ - goto l227 - l230: - position, tokenIndex = position227, tokenIndex227 - if buffer[position] != rune('_') { goto l231 } position++ - goto l227 + goto l230 l231: - position, tokenIndex = position227, tokenIndex227 + position, tokenIndex = position230, tokenIndex230 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l232 + } + position++ + goto l230 + l232: + position, tokenIndex = position230, tokenIndex230 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l233 + } + position++ + goto l230 + l233: + position, tokenIndex = position230, tokenIndex230 + if buffer[position] != rune('_') { + goto l234 + } + position++ + goto l230 + l234: + position, tokenIndex = position230, tokenIndex230 if buffer[position] != rune('-') { - goto l226 + goto l229 } position++ } - l227: - goto l225 - l226: - position, tokenIndex = position226, tokenIndex226 + l230: + goto l228 + l229: + position, tokenIndex = position229, tokenIndex229 } - add(rulefieldExpr, position222) + add(rulefieldExpr, position225) } return true - l221: - position, tokenIndex = position221, tokenIndex221 + l224: + position, tokenIndex = position224, tokenIndex224 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action40)> */ + /* 17 field <- <(<(fieldExpr / reserved)> Action42)> */ func() bool { - position232, tokenIndex232 := position, tokenIndex + position235, tokenIndex235 := position, tokenIndex { - position233 := position + position236 := position { - position234 := position + position237 := position { - position235, tokenIndex235 := position, tokenIndex + position238, tokenIndex238 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l236 + goto l239 } - goto l235 - l236: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l239: + position, tokenIndex = position238, tokenIndex238 { - position237 := position + position240 := position { - position238, tokenIndex238 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex if buffer[position] != rune('_') { - goto l239 + goto l242 } position++ if buffer[position] != rune('r') { - goto l239 + goto l242 } position++ if buffer[position] != rune('o') { - goto l239 + goto l242 } position++ if buffer[position] != rune('w') { - goto l239 + goto l242 } position++ - goto l238 - l239: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l242: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l240 + goto l243 } position++ if buffer[position] != rune('c') { - goto l240 + goto l243 } position++ if buffer[position] != rune('o') { - goto l240 + goto l243 } position++ if buffer[position] != rune('l') { - goto l240 + goto l243 } position++ - goto l238 - l240: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l243: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l241 + goto l244 } position++ if buffer[position] != rune('s') { - goto l241 + goto l244 } position++ if buffer[position] != rune('t') { - goto l241 + goto l244 } position++ if buffer[position] != rune('a') { - goto l241 + goto l244 } position++ if buffer[position] != rune('r') { - goto l241 + goto l244 } position++ if buffer[position] != rune('t') { - goto l241 + goto l244 } position++ - goto l238 - l241: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l244: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l242 + goto l245 } position++ if buffer[position] != rune('e') { - goto l242 + goto l245 } position++ if buffer[position] != rune('n') { - goto l242 + goto l245 } position++ if buffer[position] != rune('d') { - goto l242 + goto l245 } position++ - goto l238 - l242: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l245: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l243 + goto l246 } position++ if buffer[position] != rune('t') { - goto l243 + goto l246 } position++ if buffer[position] != rune('i') { - goto l243 + goto l246 } position++ if buffer[position] != rune('m') { - goto l243 + goto l246 } position++ if buffer[position] != rune('e') { - goto l243 + goto l246 } position++ if buffer[position] != rune('s') { - goto l243 + goto l246 } position++ if buffer[position] != rune('t') { - goto l243 + goto l246 } position++ if buffer[position] != rune('a') { - goto l243 + goto l246 } position++ if buffer[position] != rune('m') { - goto l243 + goto l246 } position++ if buffer[position] != rune('p') { - goto l243 + goto l246 } position++ - goto l238 - l243: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l246: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l232 + goto l235 } position++ if buffer[position] != rune('f') { - goto l232 + goto l235 } position++ if buffer[position] != rune('i') { - goto l232 + goto l235 } position++ if buffer[position] != rune('e') { - goto l232 + goto l235 } position++ if buffer[position] != rune('l') { - goto l232 + goto l235 } position++ if buffer[position] != rune('d') { - goto l232 + goto l235 } position++ } - l238: - add(rulereserved, position237) + l241: + add(rulereserved, position240) } } - l235: - add(rulePegText, position234) + l238: + add(rulePegText, position237) } { - add(ruleAction40, position) + add(ruleAction42, position) } - add(rulefield, position233) + add(rulefield, position236) } return true - l232: - position, tokenIndex = position232, tokenIndex232 + l235: + position, tokenIndex = position235, tokenIndex235 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action41)> */ + /* 19 posfield <- <( Action43)> */ func() bool { - position246, tokenIndex246 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position247 := position + position250 := position { - position248 := position + position251 := position if !_rules[rulefieldExpr]() { - goto l246 + goto l249 } - add(rulePegText, position248) + add(rulePegText, position251) } { - add(ruleAction41, position) + add(ruleAction43, position) } - add(ruleposfield, position247) + add(ruleposfield, position250) } return true - l246: - position, tokenIndex = position246, tokenIndex246 + l249: + position, tokenIndex = position249, tokenIndex249 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position250, tokenIndex250 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position251 := position + position254 := position { - position252, tokenIndex252 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l256 + } + position++ + l257: + { + position258, tokenIndex258 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l258 + } + position++ + goto l257 + l258: + position, tokenIndex = position258, tokenIndex258 + } + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 + if buffer[position] != rune('0') { goto l253 } position++ - l254: - { - position255, tokenIndex255 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l255 - } - position++ - goto l254 - l255: - position, tokenIndex = position255, tokenIndex255 - } - goto l252 - l253: - position, tokenIndex = position252, tokenIndex252 - if buffer[position] != rune('0') { - goto l250 - } - position++ } - l252: - add(ruleuint, position251) + l255: + add(ruleuint, position254) } return true - l250: - position, tokenIndex = position250, tokenIndex250 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, - /* 21 uintrow <- <( Action42)> */ + /* 21 uintrow <- <( Action44)> */ nil, - /* 22 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ + /* 22 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ func() bool { - position257, tokenIndex257 := position, tokenIndex + position260, tokenIndex260 := position, tokenIndex { - position258 := position + position261 := position { - position259, tokenIndex259 := position, tokenIndex - { - position261 := position - if !_rules[ruleuint]() { - goto l260 - } - add(rulePegText, position261) - } - { - add(ruleAction43, position) - } - goto l259 - l260: - position, tokenIndex = position259, tokenIndex259 - if buffer[position] != rune('\'') { - goto l263 - } - position++ + position262, tokenIndex262 := position, tokenIndex { position264 := position - if !_rules[rulesinglequotedstring]() { + if !_rules[ruleuint]() { goto l263 } add(rulePegText, position264) } - if buffer[position] != rune('\'') { - goto l263 - } - position++ - { - add(ruleAction44, position) - } - goto l259 - l263: - position, tokenIndex = position259, tokenIndex259 - if buffer[position] != rune('"') { - goto l257 - } - position++ - { - position266 := position - if !_rules[ruledoublequotedstring]() { - goto l257 - } - add(rulePegText, position266) - } - if buffer[position] != rune('"') { - goto l257 - } - position++ { add(ruleAction45, position) } + goto l262 + l263: + position, tokenIndex = position262, tokenIndex262 + if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + position267 := position + if !_rules[rulesinglequotedstring]() { + goto l266 + } + add(rulePegText, position267) + } + if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + add(ruleAction46, position) + } + goto l262 + l266: + position, tokenIndex = position262, tokenIndex262 + if buffer[position] != rune('"') { + goto l260 + } + position++ + { + position269 := position + if !_rules[ruledoublequotedstring]() { + goto l260 + } + add(rulePegText, position269) + } + if buffer[position] != rune('"') { + goto l260 + } + position++ + { + add(ruleAction47, position) + } } - l259: - add(rulecol, position258) + l262: + add(rulecol, position261) } return true - l257: - position, tokenIndex = position257, tokenIndex257 + l260: + position, tokenIndex = position260, tokenIndex260 return false }, - /* 23 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ + /* 23 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ nil, /* 24 open <- <('(' sp)> */ func() bool { - position269, tokenIndex269 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex { - position270 := position + position273 := position if buffer[position] != rune('(') { - goto l269 + goto l272 } position++ if !_rules[rulesp]() { - goto l269 + goto l272 } - add(ruleopen, position270) + add(ruleopen, position273) } return true - l269: - position, tokenIndex = position269, tokenIndex269 + l272: + position, tokenIndex = position272, tokenIndex272 return false }, /* 25 close <- <(')' sp)> */ func() bool { - position271, tokenIndex271 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position272 := position + position275 := position if buffer[position] != rune(')') { - goto l271 + goto l274 } position++ if !_rules[rulesp]() { - goto l271 + goto l274 } - add(ruleclose, position272) + add(ruleclose, position275) } return true - l271: - position, tokenIndex = position271, tokenIndex271 + l274: + position, tokenIndex = position274, tokenIndex274 return false }, /* 26 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position274 := position - l275: + position277 := position + l278: { - position276, tokenIndex276 := position, tokenIndex + position279, tokenIndex279 := position, tokenIndex { - position277, tokenIndex277 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex if buffer[position] != rune(' ') { - goto l278 + goto l281 } position++ - goto l277 - l278: - position, tokenIndex = position277, tokenIndex277 + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 if buffer[position] != rune('\t') { + goto l282 + } + position++ + goto l280 + l282: + position, tokenIndex = position280, tokenIndex280 + if buffer[position] != rune('\n') { goto l279 } position++ - goto l277 - l279: - position, tokenIndex = position277, tokenIndex277 - if buffer[position] != rune('\n') { - goto l276 - } - position++ } - l277: - goto l275 - l276: - position, tokenIndex = position276, tokenIndex276 + l280: + goto l278 + l279: + position, tokenIndex = position279, tokenIndex279 } - add(rulesp, position274) + add(rulesp, position277) } return true }, /* 27 comma <- <(sp ',' sp)> */ func() bool { - position280, tokenIndex280 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position281 := position + position284 := position if !_rules[rulesp]() { - goto l280 + goto l283 } if buffer[position] != rune(',') { - goto l280 + goto l283 } position++ if !_rules[rulesp]() { - goto l280 + goto l283 } - add(rulecomma, position281) + add(rulecomma, position284) } return true - l280: - position, tokenIndex = position280, tokenIndex280 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, /* 28 lbrack <- <('[' sp)> */ @@ -2740,139 +2792,139 @@ func (p *PQL) Init() { nil, /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position285, tokenIndex285 := position, tokenIndex + position288, tokenIndex288 := position, tokenIndex { - position286 := position + position289 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('-') { - goto l285 + goto l288 } position++ { - position287, tokenIndex287 := position, tokenIndex + position290, tokenIndex290 := position, tokenIndex if buffer[position] != rune('0') { + goto l291 + } + position++ + goto l290 + l291: + position, tokenIndex = position290, tokenIndex290 + if buffer[position] != rune('1') { goto l288 } position++ - goto l287 - l288: - position, tokenIndex = position287, tokenIndex287 - if buffer[position] != rune('1') { - goto l285 - } - position++ } - l287: + l290: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('-') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('T') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune(':') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ - add(ruletimestampbasicfmt, position286) + add(ruletimestampbasicfmt, position289) } return true - l285: - position, tokenIndex = position285, tokenIndex285 + l288: + position, tokenIndex = position288, tokenIndex288 return false }, /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position289, tokenIndex289 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex { - position290 := position + position293 := position { - position291, tokenIndex291 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex if buffer[position] != rune('"') { - goto l292 + goto l295 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l292 + goto l295 } if buffer[position] != rune('"') { + goto l295 + } + position++ + goto l294 + l295: + position, tokenIndex = position294, tokenIndex294 + if buffer[position] != rune('\'') { + goto l296 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l296 + } + if buffer[position] != rune('\'') { + goto l296 + } + position++ + goto l294 + l296: + position, tokenIndex = position294, tokenIndex294 + if !_rules[ruletimestampbasicfmt]() { goto l292 } - position++ - goto l291 - l292: - position, tokenIndex = position291, tokenIndex291 - if buffer[position] != rune('\'') { - goto l293 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l293 - } - if buffer[position] != rune('\'') { - goto l293 - } - position++ - goto l291 - l293: - position, tokenIndex = position291, tokenIndex291 - if !_rules[ruletimestampbasicfmt]() { - goto l289 - } } - l291: - add(ruletimestampfmt, position290) + l294: + add(ruletimestampfmt, position293) } return true - l289: - position, tokenIndex = position289, tokenIndex289 + l292: + position, tokenIndex = position292, tokenIndex292 return false }, - /* 33 timestamp <- <( Action49)> */ + /* 33 timestamp <- <( Action51)> */ nil, /* 35 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2894,86 +2946,90 @@ func (p *PQL) Init() { nil, /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 45 Action10 <- <{p.startCall("TopN")}> */ + /* 45 Action10 <- <{p.startCall("Store")}> */ nil, /* 46 Action11 <- <{p.endCall()}> */ nil, - /* 47 Action12 <- <{p.startCall("Range")}> */ + /* 47 Action12 <- <{p.startCall("TopN")}> */ nil, /* 48 Action13 <- <{p.endCall()}> */ nil, + /* 49 Action14 <- <{p.startCall("Range")}> */ nil, - /* 50 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 50 Action15 <- <{p.endCall()}> */ nil, - /* 51 Action15 <- <{ p.endCall() }> */ nil, - /* 52 Action16 <- <{ p.addBTWN() }> */ + /* 52 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 53 Action17 <- <{ p.addLTE() }> */ + /* 53 Action17 <- <{ p.endCall() }> */ nil, - /* 54 Action18 <- <{ p.addGTE() }> */ + /* 54 Action18 <- <{ p.addBTWN() }> */ nil, - /* 55 Action19 <- <{ p.addEQ() }> */ + /* 55 Action19 <- <{ p.addLTE() }> */ nil, - /* 56 Action20 <- <{ p.addNEQ() }> */ + /* 56 Action20 <- <{ p.addGTE() }> */ nil, - /* 57 Action21 <- <{ p.addLT() }> */ + /* 57 Action21 <- <{ p.addEQ() }> */ nil, - /* 58 Action22 <- <{ p.addGT() }> */ + /* 58 Action22 <- <{ p.addNEQ() }> */ nil, - /* 59 Action23 <- <{p.startConditional()}> */ + /* 59 Action23 <- <{ p.addLT() }> */ nil, - /* 60 Action24 <- <{p.endConditional()}> */ + /* 60 Action24 <- <{ p.addGT() }> */ nil, - /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action25 <- <{p.startConditional()}> */ nil, - /* 62 Action26 <- <{p.condAdd(buffer[begin:end])}> */ + /* 62 Action26 <- <{p.endConditional()}> */ nil, /* 63 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action28 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 64 Action28 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 65 Action29 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 65 Action29 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 66 Action30 <- <{ p.startList() }> */ + /* 66 Action30 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 67 Action31 <- <{ p.endList() }> */ + /* 67 Action31 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 68 Action32 <- <{ p.addVal(nil) }> */ + /* 68 Action32 <- <{ p.startList() }> */ nil, - /* 69 Action33 <- <{ p.addVal(true) }> */ + /* 69 Action33 <- <{ p.endList() }> */ nil, - /* 70 Action34 <- <{ p.addVal(false) }> */ + /* 70 Action34 <- <{ p.addVal(nil) }> */ nil, - /* 71 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addVal(true) }> */ nil, - /* 72 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addVal(false) }> */ nil, - /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 74 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, /* 75 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action40 <- <{ p.addField(buffer[begin:end]) }> */ + /* 76 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action41 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 77 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 78 Action42 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 78 Action42 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 79 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 79 Action43 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 80 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 80 Action44 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 81 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 81 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 82 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 82 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 83 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 84 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 84 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 85 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 85 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 86 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 87 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/server.go b/server.go index 61e4838a1..70391723e 100644 --- a/server.go +++ b/server.go @@ -481,7 +481,7 @@ func (s *Server) receiveMessage(m Message) error { if f == nil { return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field) } - if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { + if err := f.AddRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { return errors.Wrap(err, "adding remote available shards") } case *CreateIndexMessage: @@ -509,6 +509,11 @@ func (s *Server) receiveMessage(m Message) error { if err := idx.DeleteField(obj.Field); err != nil { return err } + case *DeleteAvailableShardMessage: + f := s.holder.Field(obj.Index, obj.Field) + if err := f.RemoveAvailableShard(obj.ShardID); err != nil { + return err + } case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { @@ -648,7 +653,7 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name) continue } - if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil { + if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { return errors.Wrap(err, "adding remote available shards") } } @@ -675,6 +680,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.nodeID) s.diagnostics.Set("ClusterID", s.cluster.id) + s.diagnostics.EnrichWithCPUInfo() s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval diff --git a/server/handler_test.go b/server/handler_test.go index 2c1aa9868..9e76d77b9 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) { }) + t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) { + // Roaring import into a non-set field should fail. + if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData)) + req.Header.Set("Content-Type", "application/x-binary") + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + }) + t.Run("Status", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) diff --git a/server/server.go b/server/server.go index b4ac595fa..010eb2f8a 100644 --- a/server/server.go +++ b/server/server.go @@ -364,7 +364,10 @@ func (m *Command) Close() error { eg.Go(m.gossipMemberSet.Close) } if closer, ok := m.logOutput.(io.Closer); ok { - eg.Go(closer.Close) + // If closer is os.Stdout or os.Stderr, don't close it. + if closer != os.Stdout && closer != os.Stderr { + eg.Go(closer.Close) + } } err := eg.Wait() return errors.Wrap(err, "closing everything")