diff --git a/Makefile b/Makefile index e7af8e119..55648c7f1 100644 --- a/Makefile +++ b/Makefile @@ -139,9 +139,9 @@ gometalinter: require-gometalinter --enable=ineffassign \ --enable=interfacer \ --enable=maligned \ - --enable=megacheck \ --enable=misspell \ --enable=nakedret \ + --enable=staticcheck \ --enable=unconvert \ --enable=unparam \ --enable=vet \ diff --git a/cluster.go b/cluster.go index e966c57a1..4565e91f3 100644 --- a/cluster.go +++ b/cluster.go @@ -609,8 +609,8 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool { // Nodes returns a copy of the slice of nodes in the cluster. Safe for // concurrent use, result may be modified. func (c *cluster) Nodes() []*Node { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() ret := make([]*Node, len(c.nodes)) copy(ret, c.nodes) return ret @@ -1792,7 +1792,7 @@ func (c *cluster) nodeLeave(nodeID string) error { } if c.state != ClusterStateNormal && c.state != ClusterStateDegraded { - return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", + return fmt.Errorf("cluster must be '%s' to remove a node but is '%s'", ClusterStateNormal, c.state) } @@ -1803,7 +1803,7 @@ func (c *cluster) nodeLeave(nodeID string) error { // Prevent removing the coordinator node (this node). if nodeID == c.Node.ID { - return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.") + return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator") } // See if resize job can be generated diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 50602d385..a6cf437b2 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -83,7 +83,7 @@ func TestFragCombos(t *testing.T) { // newIndexWithTempPath returns a new instance of Index. func newIndexWithTempPath(name string) *Index { - path, err := ioutil.TempDir("", "pilosa-index-") + path, err := ioutil.TempDir(*TempDir, "pilosa-index-") if err != nil { panic(err) } diff --git a/ctl/import.go b/ctl/import.go index ab19064ce..c120c955f 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -151,11 +151,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) if err != nil { - return fmt.Errorf("Error Creating Index: %s", err) + return errors.Wrap(err, "creating index") } err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.FieldOptions) if err != nil { - return fmt.Errorf("Error Creating Field: %s", err) + return errors.Wrap(err, "creating field") } return nil } diff --git a/diagnostics.go b/diagnostics.go index 5ed97940a..1eb3f2606 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -137,11 +137,11 @@ func (d *diagnosticsCollector) compareVersion(value string) error { localVersion := versionSegments(d.version) if localVersion[0] < currentVersion[0] { //Major - return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) + return fmt.Errorf("you are running Pilosa %s, a newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor - return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) + return fmt.Errorf("you are running Pilosa %s, the latest minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch - return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) + return fmt.Errorf("there is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) } return nil diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index f1536e1d1..99c0a83ac 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -82,19 +82,19 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { d.SetVersion(version) err := d.compareVersion("v1.7.0") - if !strings.Contains(err.Error(), "A newer version") { + if !strings.Contains(err.Error(), "a newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.compareVersion("1.7.0") - if !strings.Contains(err.Error(), "A newer version") { + if !strings.Contains(err.Error(), "a newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.compareVersion("0.7.0") - if !strings.Contains(err.Error(), "The latest Minor release is") { + if !strings.Contains(err.Error(), "the latest minor release is") { t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) } err = d.compareVersion("0.1.2") - if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") { + if !strings.Contains(err.Error(), "there is a new patch release of Pilosa") { t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) } err = d.compareVersion("0.1.1") diff --git a/docs/configuration.md b/docs/configuration.md index 140258437..a8dafa91d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -366,7 +366,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -379,7 +379,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -392,7 +392,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -410,7 +410,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -428,7 +428,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -446,7 +446,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -468,7 +468,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12000 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -486,7 +486,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12001 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -504,7 +504,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12002 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] diff --git a/docs/query-language.md b/docs/query-language.md index 73e1cab7d..82ffecd2a 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -786,7 +786,7 @@ Options(Row(f1=10), shards=[0, 2]) **Spec:** ``` -Rows(field=, previous=, limit=, column=) +Rows(, previous=, limit=, column=) ``` **Description:** @@ -810,7 +810,7 @@ result of the previous request will start from the next available row. Without keys: ```request -Rows(field=blah) +Rows(blah) ``` ```response {"rows":[1,9,39]} @@ -818,7 +818,7 @@ Rows(field=blah) With keys: ```request -Rows(field=blahk) +Rows(blahk) ``` ```response {"rows":null,"keys":["haha","zaaa","traa"]} @@ -859,7 +859,7 @@ specify the field and row for each row that was intersected to get that result. A single `Rows` query. ```request -GroupBy(Rows(field=blah)) +GroupBy(Rows(blah)) ``` ```response [{"group":[{"field":"blah","rowID":1}],"count":1}, @@ -869,7 +869,7 @@ GroupBy(Rows(field=blah)) With two `Rows` queries - one with IDs and one with keys. ```request -GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) +GroupBy(Rows(blah), Rows(blahk), limit=7) ``` ```response [{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1}, @@ -883,7 +883,7 @@ GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) Getting the rest of the results from the previous example (paging). ```request -GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7) +GroupBy(Rows(blah, previous=39), Rows(blahk, previous="haha"), limit=7) ``` ```response diff --git a/executor.go b/executor.go index 28290288c..b765a5ac9 100644 --- a/executor.go +++ b/executor.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "log" "sort" "time" @@ -808,7 +807,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca return nil, nil } - if minThreshold <= 0 { + if minThreshold == 0 { minThreshold = defaultMinThreshold } @@ -917,6 +916,12 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // TODO support TopN in here would be really cool - and pretty easy I think. childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { + // Check "field" first for backwards compatibility, then set _field. + // TODO: remove at Pilosa 2.0 + if fieldName, ok := child.Args["field"].(string); ok { + child.Args["_field"] = fieldName + } + if child.Name != "Rows" { return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name) } @@ -1091,6 +1096,17 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + // Fetch field name from argument. + // Check "field" first for backwards compatibility. + // TODO: remove at Pilosa 2.0 + var fieldName string + var ok bool + if fieldName, ok = c.Args["field"].(string); ok { + c.Args["_field"] = fieldName + } + if fieldName, ok = c.Args["_field"].(string); !ok { + return nil, errors.New("Rows() field required") + } if columnID, ok, err := c.UintArg("column"); err != nil { return nil, errors.Wrap(err, "getting column") } else if ok { @@ -1099,7 +1115,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeRowsShard(ctx, index, c, shard) + return e.executeRowsShard(ctx, index, fieldName, c, shard) } // Determine limit so we can use it when reducing. @@ -1124,22 +1140,25 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - // Fetch field name from argument. - fieldName, ok := c.Args["field"].(string) - if !ok { - return nil, errors.New("Rows() argument required: field") - } // Fetch field. f := e.Holder.Field(index, fieldName) if f == nil { return nil, ErrFieldNotFound } + + // Rows query does not currently support a `time` field that has + // `noStandardView: true`. + // TODO https://github.com/pilosa/pilosa/issues/1783 + if f.Type() == FieldTypeTime && f.options.NoStandardView { + return nil, errors.New("Rows() query on time field with no standard view is not currently supported") + } + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { return make(RowIDs, 0), nil @@ -1176,7 +1195,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal defer span.Finish() if c.Name == "Range" { - log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.") + e.Holder.Logger.Printf("DEPRECATED: Range() is deprecated, please use Row() instead.") } // Handle bsiGroup ranges differently. @@ -1586,14 +1605,14 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if err != nil { return false, fmt.Errorf("reading Clear() row: %v", err) } else if !ok { - return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel) + return false, fmt.Errorf("row= argument required to Clear() call") } colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { return false, fmt.Errorf("reading Clear() column: %v", err) } else if !ok { - return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel) + return false, fmt.Errorf("column argument to Clear(, =) required") } return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) @@ -1712,19 +1731,19 @@ func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pq return changed, nil } -// executeSetRow executes a SetRow() call. +// executeSetRow executes a Store() call. func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { - // Ensure the field type supports SetRow(). + // Ensure the field type supports Store(). fieldName, err := c.FieldArg() if err != nil { - return false, errors.New("SetRow() argument required: field") + return false, errors.New("field required for Store()") } field := e.Holder.Field(index, fieldName) if field == nil { return false, ErrFieldNotFound } if field.Type() != FieldTypeSet { - return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) + return false, fmt.Errorf("can't Store() on a %s field", field.Type()) } // Execute calls in bulk on each remote node and merge. @@ -1749,15 +1768,15 @@ func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { fieldName, err := c.FieldArg() if err != nil { - return false, errors.New("SetRow() argument required: field") + return false, errors.New("Store() argument required: field") } // Read fields using labels. rowID, ok, err := c.UintArg(fieldName) if err != nil { - return false, fmt.Errorf("reading SetRow() row: %v", err) + return false, fmt.Errorf("reading Store() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel) + return false, fmt.Errorf("need the = argument on Store()") } field := e.Holder.Field(index, fieldName) @@ -1774,7 +1793,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. } src = row } else { - return false, errors.New("SetRow() requires a source row") + return false, errors.New("Store() requires a source row") } // Set the row on the standard view. @@ -1793,7 +1812,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. } set, err := fragment.setRow(src, rowID) if err != nil { - return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) + return false, errors.Wrapf(err, "storing row %d on view %s shard %d", rowID, viewStandard, shard) } changed = changed || set @@ -2354,7 +2373,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") case "Rows": - fieldName = callArgString(c, "field") + fieldName = callArgString(c, "_field") rowKey = "previous" colKey = "column" case "GroupBy": @@ -2459,7 +2478,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e fields := make([]*Field, len(c.Children)) for i, child := range c.Children { - fieldname := callArgString(child, "field") + fieldname := callArgString(child, "_field") field := idx.Field(fieldname) if field == nil { return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) @@ -2570,7 +2589,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res case RowIDs: other := RowIdentifiers{} - fieldName := callArgString(call, "field") + fieldName := callArgString(call, "_field") if fieldName == "" { return nil, ErrFieldNotFound } @@ -2768,11 +2787,12 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde fields: make([]FieldRow, len(children)), } + var fieldName string + var ok bool ignorePrev := false for i, call := range children { - fieldName, ok := call.Args["field"].(string) - if !ok { - return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"]) + if fieldName, ok = call.Args["_field"].(string); !ok { + return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["_field"]) } if holder.Field(index, fieldName) == nil { return nil, ErrFieldNotFound diff --git a/executor_internal_test.go b/executor_internal_test.go index 629d704db..4177968d5 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -14,7 +14,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { e := &executor{ Holder: NewHolder(), } - e.Holder.Path, _ = ioutil.TempDir("", "") + e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") err := e.Holder.Open() if err != nil { t.Fatalf("opening holder: %v", err) @@ -46,7 +46,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("translating rows %v, %v", erra, errb) } - query, err := pql.ParseString(`GroupBy(Rows(field=ak), Rows(field=b), Rows(field=ck), previous=["la", 0, "ha"])`) + query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`) if err != nil { t.Fatalf("parsing query: %v", err) } @@ -69,28 +69,28 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { err string }{ { - pql: `GroupBy(Rows(field=notfound), previous=1)`, + pql: `GroupBy(Rows(notfound), previous=1)`, err: "'previous' argument must be list", }, { - pql: `GroupBy(Rows(field=ak), previous=["la", 0])`, + pql: `GroupBy(Rows(ak), previous=["la", 0])`, err: "mismatched lengths", }, { - pql: `GroupBy(Rows(field=ak), previous=[1])`, + pql: `GroupBy(Rows(ak), previous=[1])`, err: "prev value must be a string", }, { - pql: `GroupBy(Rows(field=notfound), previous=[1])`, + pql: `GroupBy(Rows(notfound), previous=[1])`, err: ErrFieldNotFound.Error(), }, // TODO: an unknown key will actually allocate an id. this is probably bad. // { - // pql: `GroupBy(Rows(field=ak), previous=["zoop"])`, + // pql: `GroupBy(Rows(ak), previous=["zoop"])`, // err: "translating row key '", // }, { - pql: `GroupBy(Rows(field=b), previous=["la"])`, + pql: `GroupBy(Rows(b), previous=["la"])`, err: "which doesn't use string keys", }, } diff --git a/executor_test.go b/executor_test.go index 2d155bd97..47b8ec9b3 100644 --- a/executor_test.go +++ b/executor_test.go @@ -16,7 +16,9 @@ package pilosa_test import ( "context" + "flag" "fmt" + "io/ioutil" "math/rand" "reflect" "strconv" @@ -33,6 +35,22 @@ import ( "github.com/pkg/errors" ) +var ( + TempDir = getTempDirString() +) + +func getTempDirString() (td *string) { + tdflag := flag.Lookup("temp-dir") + + if tdflag == nil { + td = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") + } else { + s := tdflag.Value.String() + td = &s + } + return td +} + // Ensure a row query can be executed. func TestExecutor_Execute_Row(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { @@ -2283,7 +2301,7 @@ Set(4500001, fn=4) t.Run("remote groupBy", func(t *testing.T) { if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", - Query: `GroupBy(Rows(field=f))`, + Query: `GroupBy(Rows(f))`, }); err != nil { t.Fatalf("GroupBy querying: %v", err) } else { @@ -2987,7 +3005,16 @@ func TestExecutor_Execute_SetRow(t *testing.T) { } func benchmarkExistence(nn bool, b *testing.B) { - c := test.MustRunCluster(b, 1) + c := test.MustNewCluster(b, 1) + var err error + c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkExistence") + if err != nil { + b.Fatalf("getting temp dir: %v", err) + } + err = c.Start() + if err != nil { + b.Fatalf("starting cluster: %v", err) + } defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -3038,27 +3065,45 @@ func TestExecutor_Execute_Rows(t *testing.T) { {13, 3}, }) - rows := c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) + rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, limit=2)`).Results[0].(pilosa.RowIdentifiers) + // backwards compatibility + // TODO: remove at Pilosa 2.0 + rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { + t.Fatalf("unexpected rows: %+v", rows) + } + + rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) + rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, column=2)`).Results[0].(pilosa.RowIdentifiers) + rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } } +func TestExecutor_Execute_RowsTime(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + + exp := "executing: Rows() query on time field with no standard view is not currently supported" + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp { + t.Fatalf("expected error: %s", exp) + } +} + func TestExecutor_Execute_Query_Error(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -3070,35 +3115,27 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { }{ { query: "GroupBy(Rows())", - error: "Rows call must have 'field' argument", + error: "Rows call must have field", }, { - query: "GroupBy(Rows(field=true))", - error: "Rows call must have 'field' argument", + query: "GroupBy(Rows(\"true\"))", + error: "parsing: parsing:", }, { - query: "GroupBy(Rows(field=\"true\"))", - error: "field not found", + query: "GroupBy(Rows(1))", + error: "parsing: parsing:", }, { - query: "GroupBy(Rows(field=1))", - error: "Rows call must have 'field' argument", - }, - { - query: "GroupBy(Rows(field))", - error: "parse error", - }, - { - query: "GroupBy(Rows(field=general, limit=-1))", + query: "GroupBy(Rows(general, limit=-1))", error: "must be positive, but got", }, { - query: "GroupBy(Rows(field=general), limit=-1)", + query: "GroupBy(Rows(general), limit=-1)", error: "must be positive, but got", }, { - query: "GroupBy(Rows(field=general), filter=Rows(field=general))", - error: "unknown call: Rows", + query: "GroupBy(Rows(general), filter=Rows(general))", + error: "parsing: parsing:", }, } @@ -3112,7 +3149,7 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r) } if !strings.Contains(err.Error(), test.error) { - t.Fatalf("unexpected error message: %s", err.Error()) + t.Fatalf("unexpected error message:\n%s != %s", test.error, err.Error()) } }) } @@ -3158,68 +3195,80 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { q string exp []string }{ + { + q: `Rows(f)`, + exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"}, + }, + // backwards compatibility + // TODO: remove at Pilosa 2.0 { q: `Rows(field=f)`, exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"}, }, + { + q: `Rows(f, limit=2)`, + exp: []string{"0", "1"}, + }, + // backwards compatibility + // TODO: remove at Pilosa 2.0 { q: `Rows(field=f, limit=2)`, exp: []string{"0", "1"}, }, { - q: `Rows(field=f, previous="15")`, + q: `Rows(f, previous="15")`, exp: []string{"16", "17", "18"}, }, { - q: `Rows(field=f, previous="11", limit=2)`, + q: `Rows(f, previous="11", limit=2)`, exp: []string{"12", "13"}, }, { - q: `Rows(field=f, previous="17", limit=5)`, + q: `Rows(f, previous="17", limit=5)`, exp: []string{"18"}, }, { - q: `Rows(field=f, previous="18")`, + q: `Rows(f, previous="18")`, exp: []string{}, }, { - q: `Rows(field=f, previous="1", limit=0)`, + q: `Rows(f, previous="1", limit=0)`, exp: []string{}, }, { - q: `Rows(field=f, column="1")`, + q: `Rows(f, column="1")`, exp: []string{"0", "1"}, }, { - q: `Rows(field=f, column="2")`, + q: `Rows(f, column="2")`, exp: []string{"0", "1", "2"}, }, { - q: `Rows(field=f, column="3")`, + q: `Rows(f, column="3")`, exp: []string{"1", "2", "3"}, }, { - q: `Rows(field=f, limit=2, column="3")`, + q: `Rows(f, limit=2, column="3")`, exp: []string{"1", "2"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="15", column="%d")`, ShardWidth*9+17), + q: fmt.Sprintf(`Rows(f, previous="15", column="%d")`, ShardWidth*9+17), exp: []string{"16", "17"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column="%d")`, ShardWidth*5+14), + q: fmt.Sprintf(`Rows(f, previous="11", limit=2, column="%d")`, ShardWidth*5+14), exp: []string{"12", "13"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column="%d")`, ShardWidth*9+18), + q: fmt.Sprintf(`Rows(f, previous="17", limit=5, column="%d")`, ShardWidth*9+18), exp: []string{"18"}, }, { - q: `Rows(field=f, previous="18", column="19")`, + q: `Rows(f, previous="18", column="19")`, exp: []string{}, }, { - q: `Rows(field=f, previous="1", limit=0, column="0")`, + q: `Rows(f, previous="1", limit=0, column="0")`, exp: []string{}, }, } @@ -3271,13 +3320,27 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("Unknown Field ", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(missing))`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) } } }) + // backwards compatibility + // TODO: remove at Pilosa 2.0 + t.Run("BasicLegacy", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(sub))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + t.Run("Basic", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, @@ -3286,7 +3349,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3296,7 +3359,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3306,7 +3369,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3315,7 +3378,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3336,7 +3399,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3364,7 +3427,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test wrapping with previous", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, @@ -3374,14 +3437,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test previous is last result", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa, previous=3), Rows(field=wb, previous=3), Rows(field=wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) if len(results) > 0 { t.Fatalf("expected no results because previous specified last result") } }) t.Run("test wrapping multiple", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } @@ -3405,7 +3468,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {3, ShardWidth}, }) t.Run("distinct rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, @@ -3417,7 +3480,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, @@ -3428,7 +3491,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { - results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(field=ma), Rows(field=mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, @@ -3453,7 +3516,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {1, ShardWidth}, }) t.Run("same rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=na), Rows(field=nb))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2}, @@ -3490,11 +3553,11 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("test wrapping with previous", func(t *testing.T) { totalResults := make([]pilosa.GroupCount, 0) - results := c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount) totalResults = append(totalResults, results...) for len(totalResults) < 64 { lastGroup := results[len(results)-1].Group - query := fmt.Sprintf("GroupBy(Rows(field=ppa, previous=%d), Rows(field=ppb, previous=%d), Rows(field=ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) + query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount) totalResults = append(totalResults, results...) } @@ -3537,7 +3600,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3551,7 +3614,16 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } func BenchmarkGroupBy(b *testing.B) { - c := test.MustRunCluster(b, 1) + c := test.MustNewCluster(b, 1) + var err error + c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkGroupBy") + if err != nil { + b.Fatalf("getting temp dir: %v", err) + } + err = c.Start() + if err != nil { + b.Fatalf("starting cluster: %v", err) + } defer c.Close() c.CreateField(b, "i", pilosa.IndexOptions{}, "a") c.CreateField(b, "i", pilosa.IndexOptions{}, "b") @@ -3586,7 +3658,7 @@ func BenchmarkGroupBy(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c))`) + c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c))`) } }) @@ -3594,7 +3666,7 @@ func BenchmarkGroupBy(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c), limit=4)`) + c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c), limit=4)`) } }) diff --git a/field_internal_test.go b/field_internal_test.go index c2495e9c0..0a26225bb 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -192,7 +192,7 @@ type TestField struct { // NewTestField returns a new instance of TestField d/0. func NewTestField(opts FieldOption) *TestField { - path, err := ioutil.TempDir("", "pilosa-field-") + path, err := ioutil.TempDir(*TempDir, "pilosa-field-") if err != nil { panic(err) } diff --git a/fragment.go b/fragment.go index 4f71809c9..58680e655 100644 --- a/fragment.go +++ b/fragment.go @@ -1050,7 +1050,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { rowID, cnt := pair.ID, pair.Count // Ignore empty rows. - if cnt <= 0 { + if cnt == 0 { continue } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index cca88dda1..548e5f40a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -39,12 +39,9 @@ var ( // In order to generate the sample fragment file, // run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") - TempDir = "" -) -func init() { // nolint: gochecknoinits - flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") -} + TempDir = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") +) // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { @@ -367,6 +364,20 @@ func TestFragment_Sum(t *testing.T) { t.Fatalf("unexpected sum: %d", sum) } }) + + // verify that clearValue clears values + if _, err := f.clearValue(1000, bitDepth, 23); err != nil { + t.Fatal(err) + } + t.Run("ClearValue", func(t *testing.T) { + if sum, n, err := f.sum(nil, bitDepth); err != nil { + t.Fatal(err) + } else if n != 3 { + t.Fatalf("unexpected count: %d", n) + } else if sum != (3800 - 382) { + t.Fatalf("unexpected sum: got %d, expecting %d", sum, 3800-382) + } + }) } // Ensure a fragment can find the min and max of values. @@ -637,6 +648,71 @@ func TestFragment_Range(t *testing.T) { }) } +// benchmarkSetValues is a helper function to explore, very roughly, the cost +// of setting values. +func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + column := uint64(0) + for i := 0; i < b.N; i++ { + f.setValue(column, bitDepth, uint64(i)) + column = cfunc(column) + } +} + +// Benchmark performance of setValue for BSI ranges. +func BenchmarkFragment_SetValue(b *testing.B) { + depths := []uint{4, 8, 16} + for _, bitDepth := range depths { + name := fmt.Sprintf("Depth%d", bitDepth) + f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Sparse", func(b *testing.B) { + benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + }) + f.Clean(b) + f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Dense", func(b *testing.B) { + benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + }) + f.Clean(b) + } +} + +// benchmarkImportValues is a helper function to explore, very roughly, the cost +// of setting values using the special setter used for imports. +func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + column := uint64(0) + b.StopTimer() + columns := make([]uint64, b.N) + values := make([]uint64, b.N) + for i := 0; i < b.N; i++ { + values[i] = uint64(i) + columns[i] = column + column = cfunc(column) + } + b.StartTimer() + err := f.importValue(columns, values, bitDepth, false) + if err != nil { + b.Fatalf("error importing values: %s", err) + } +} + +// Benchmark performance of setValue for BSI ranges. +func BenchmarkFragment_ImportValue(b *testing.B) { + depths := []uint{4, 8, 16} + for _, bitDepth := range depths { + name := fmt.Sprintf("Depth%d", bitDepth) + f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Sparse", func(b *testing.B) { + benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + }) + f.Clean(b) + f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Dense", func(b *testing.B) { + benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + }) + f.Clean(b) + } +} + // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") @@ -1142,7 +1218,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean(b) + defer f.CleanKeep(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1671,7 +1747,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean(b) + defer f.CleanKeep(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -1990,7 +2066,7 @@ func BenchmarkFileWrite(b *testing.B) { b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, err := ioutil.TempFile(TempDir, "") + f, err := ioutil.TempFile(*TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } @@ -2030,9 +2106,23 @@ func (f *fragment) Clean(t testing.TB) { } } +// CleanKeep is just like Clean(), but it doesn't remove the +// fragment file (note that it DOES remove the cache file). +func (f *fragment) CleanKeep(t testing.TB) { + errc := f.Close() + errp := os.Remove(f.cachePath()) + if errc != nil { + t.Fatal("closing fragment: ", errc, errp) + } + // not all fragments have cache files + if errp != nil && !os.IsNotExist(errp) { + t.Fatalf("cleaning up fragment cache: %v", errp) + } +} + // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { - file, err := ioutil.TempFile(TempDir, "pilosa-fragment-") + file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") if err != nil { panic(err) } diff --git a/gossip/gossip.go b/gossip/gossip.go index 491e14cf2..e3c552f88 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "log" "net" + "os" "strconv" "strings" "sync" @@ -50,8 +51,11 @@ type memberSet struct { Logger logger.Logger - logger *log.Logger + // stdLogger is only used when passed into memberlist library things that take a std library logger rather than an interface. + stdLogger *log.Logger + // logOutput is similar to stdLogger in that it's passed to memberlist things which can't take a pilosa Logger. logOutput io.Writer + transport *Transport eventReceiver *eventReceiver @@ -151,14 +155,20 @@ func WithTransport(transport *Transport) memberSetOption { } } -// WithLogger is a functional option for providing a logger to NewMemberSet. +// WithLogger is a functional option for providing a Go logger to NewMemberSet. +// If the memberSet's transport is nil, this logger will be used when creating +// one. If WithLogOutput is not used, this logger will be passed to memberlist +// for it to use internally. This logger is not used for logging by code in this +// (gossip) package - for that, use the WithPilosaLogger option. func WithLogger(logger *log.Logger) memberSetOption { return func(g *memberSet) error { - g.logger = logger + g.stdLogger = logger return nil } } +// WithLogOutput allows one to pass a Writer which will in turn be passed to +// memberlist for use in logging. func WithLogOutput(o io.Writer) memberSetOption { return func(g *memberSet) error { g.logOutput = o @@ -166,7 +176,20 @@ func WithLogOutput(o io.Writer) memberSetOption { } } -// NewMemberSet returns a new instance of GossipMemberSet based on options. +// WithPilosaLogger allows one to configure a memberSet with a logger of their +// choice which satisfies the pilosa logger interface. +func WithPilosaLogger(l logger.Logger) memberSetOption { + return func(g *memberSet) error { + g.Logger = l + return nil + } +} + +// NewMemberSet returns a new instance of GossipMemberSet based on options. The +// logging options which can be passed to NewMemberSet are complicated for +// historical reasons - please pass WithPilosaLogger, and either WithLogOutput +// or WithLogger. If you pass WithLogOutput, be sure to also pass in a Transport +// using WithTransport. func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) { host := api.Node().URI.Host g := &memberSet{ @@ -180,7 +203,8 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem return nil, errors.Wrap(err, "executing option") } } - ger := newEventReceiver(g.logger, api) + + ger := newEventReceiver(g.Logger, api) g.eventReceiver = ger if g.transport == nil { @@ -189,8 +213,16 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem return nil, fmt.Errorf("convert port: %s", err) } + if g.stdLogger == nil { + if g.logOutput != nil { + g.stdLogger = logger.NewStandardLogger(g.logOutput).Logger() + } else { + g.stdLogger = log.New(os.Stderr, "", log.LstdFlags) + } + } + // Set up the transport. - transport, err := NewTransport(host, port, g.logger) + transport, err := NewTransport(host, port, g.stdLogger) if err != nil { return nil, fmt.Errorf("new tranport: %s", err) } @@ -233,7 +265,7 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem if g.logOutput != nil { conf.LogOutput = g.logOutput } else { - conf.Logger = g.logger + conf.Logger = g.stdLogger } g.config = &config{ @@ -318,11 +350,11 @@ type eventReceiver struct { ch chan memberlist.NodeEvent papi *pilosa.API - logger *log.Logger + logger logger.Logger } // newEventReceiver returns a new instance of GossipEventReceiver. -func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { +func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver { ger := &eventReceiver{ ch: make(chan memberlist.NodeEvent, 1), logger: logger, @@ -468,7 +500,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { nt, err := makeNetRetry(limit) if err != nil { - return nil, fmt.Errorf("Could not set up network transport: %v", err) + return nil, errors.Wrap(err, "could not set up network transport") } return nt, nil diff --git a/holder_internal_test.go b/holder_internal_test.go index 1c4dfad88..b13df7809 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -46,7 +46,7 @@ func (h *tHolder) Reopen() error { } func newHolder() *tHolder { - path, err := ioutil.TempDir("", "pilosa-") + path, err := ioutil.TempDir(*TempDir, "pilosa-") if err != nil { panic(err) } diff --git a/http/handler.go b/http/handler.go index b85be6b0e..8cc059035 100644 --- a/http/handler.go +++ b/http/handler.go @@ -584,11 +584,11 @@ func validateOptions(data map[string]interface{}, validIndexOptions []string) er } for kk, vv := range options { if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) + return fmt.Errorf("unknown key: %v:%v", kk, vv) } } default: - return fmt.Errorf("Unknown key: %v:%v", k, v) + return fmt.Errorf("unknown key: %v:%v", k, v) } } return nil diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index ecbf8ff87..0cb90655f 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -35,8 +35,8 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, - {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, + {json: `{"option": {}}`, err: "unknown key: option:map[]"}, + {json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"}, } for _, test := range tests { actual := &postIndexRequest{} diff --git a/index_internal_test.go b/index_internal_test.go index fd83fd775..dc0296049 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -21,7 +21,7 @@ import ( // mustOpenIndex returns a new, opened index at a temporary path. Panic on error. func mustOpenIndex(opt IndexOptions) *Index { - path, err := ioutil.TempDir("", "pilosa-index-") + path, err := ioutil.TempDir(*TempDir, "pilosa-index-") if err != nil { panic(err) } diff --git a/lru/lru.go b/lru/lru.go index d7bb1929c..ba0121a9a 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -83,7 +83,7 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { } // remove removes the provided key from the cache. -func (c *Cache) remove(key Key) { // nolint: megacheck +func (c *Cache) remove(key Key) { // nolint: staticcheck if c.cache == nil { return } @@ -121,7 +121,7 @@ func (c *Cache) Len() int { } // clear purges all stored items from the cache. -func (c *Cache) clear() { // nolint: megacheck +func (c *Cache) clear() { // nolint: staticcheck if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) diff --git a/pql/ast.go b/pql/ast.go index 8b23c708b..d78ad2828 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -259,7 +259,7 @@ func (c *Call) FieldArg() (string, error) { return arg, nil } } - return "", fmt.Errorf("No field argument specified") + return "", fmt.Errorf("no field argument specified") } func IsReservedArg(name string) bool { diff --git a/pql/pql.peg b/pql/pql.peg index 66c1d3265..8127ecc83 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -13,6 +13,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} + / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 8df2fcb72..8d960f95c 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -62,9 +62,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 - rulePegText ruleAction14 ruleAction15 + rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -99,6 +99,8 @@ const ( ruleAction47 ruleAction48 ruleAction49 + ruleAction50 + ruleAction51 ) var rul3s = [...]string{ @@ -149,9 +151,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", - "PegText", "Action14", "Action15", + "PegText", "Action16", "Action17", "Action18", @@ -186,6 +188,8 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", + "Action50", + "Action51", } type token32 struct { @@ -302,7 +306,7 @@ type PQL struct { Buffer string buffer []rune - rules [84]func() bool + rules [86]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -423,77 +427,81 @@ func (p *PQL) Execute() { case ruleAction13: p.endCall() case ruleAction14: - p.startCall(buffer[begin:end]) + p.startCall("Rows") case ruleAction15: p.endCall() case ruleAction16: - p.addBTWN() + p.startCall(buffer[begin:end]) case ruleAction17: - p.addLTE() + p.endCall() case ruleAction18: - p.addGTE() + p.addBTWN() case ruleAction19: - p.addEQ() + p.addLTE() case ruleAction20: - p.addNEQ() + p.addGTE() case ruleAction21: - p.addLT() + p.addEQ() case ruleAction22: - p.addGT() + p.addNEQ() case ruleAction23: - p.startConditional() + p.addLT() case ruleAction24: - p.endConditional() + p.addGT() case ruleAction25: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction26: - p.condAdd(buffer[begin:end]) + p.endConditional() case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.startList() + p.condAdd(buffer[begin:end]) case ruleAction29: - p.endList() + p.condAdd(buffer[begin:end]) case ruleAction30: - p.addVal(nil) + p.startList() case ruleAction31: - p.addVal(true) + p.endList() case ruleAction32: - p.addVal(false) + p.addVal(nil) case ruleAction33: - p.addVal(buffer[begin:end]) + p.addVal(true) case ruleAction34: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction35: - p.addNumVal(buffer[begin:end]) - case ruleAction36: - p.startCall(buffer[begin:end]) - case ruleAction37: - p.addVal(p.endCall()) - case ruleAction38: p.addVal(buffer[begin:end]) + case ruleAction36: + p.addNumVal(buffer[begin:end]) + case ruleAction37: + p.addNumVal(buffer[begin:end]) + case ruleAction38: + p.startCall(buffer[begin:end]) case ruleAction39: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(p.endCall()) case ruleAction40: p.addVal(buffer[begin:end]) case ruleAction41: - p.addField(buffer[begin:end]) + s, _ := strconv.Unquote(buffer[begin:end]) + p.addVal(s) case ruleAction42: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction43: - p.addPosNum("_col", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction44: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction45: - p.addPosStr("_col", buffer[begin:end]) + p.addPosNum("_col", buffer[begin:end]) case ruleAction46: - p.addPosNum("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction47: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction49: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -606,7 +614,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ( Action14 open allargs comma? close Action15))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'o' 'w' 's' Action14 open posfield (comma allargs)? close Action15) / ( Action16 open allargs comma? close Action17))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -655,7 +663,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction49, position) + add(ruleAction51, position) } add(ruletimestamp, position12) } @@ -741,7 +749,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction46, position) + add(ruleAction48, position) } goto l19 l20: @@ -762,7 +770,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction47, position) + add(ruleAction49, position) } goto l19 l23: @@ -783,7 +791,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction50, position) } } l19: @@ -1069,38 +1077,85 @@ func (p *PQL) Init() { goto l7 l41: position, tokenIndex = position7, tokenIndex7 - { - position46 := position - if !_rules[ruleIDENT]() { - goto l5 - } - add(rulePegText, position46) + if buffer[position] != rune('R') { + goto l46 } + position++ + if buffer[position] != rune('o') { + goto l46 + } + position++ + if buffer[position] != rune('w') { + goto l46 + } + position++ + if buffer[position] != rune('s') { + goto l46 + } + position++ { add(ruleAction14, position) } if !_rules[ruleopen]() { - goto l5 + goto l46 } - if !_rules[ruleallargs]() { - goto l5 + if !_rules[ruleposfield]() { + goto l46 } { position48, tokenIndex48 := position, tokenIndex if !_rules[rulecomma]() { goto l48 } + if !_rules[ruleallargs]() { + goto l48 + } goto l49 l48: position, tokenIndex = position48, tokenIndex48 } l49: if !_rules[ruleclose]() { - goto l5 + goto l46 } { add(ruleAction15, position) } + goto l7 + l46: + position, tokenIndex = position7, tokenIndex7 + { + position51 := position + if !_rules[ruleIDENT]() { + goto l5 + } + add(rulePegText, position51) + } + { + add(ruleAction16, position) + } + if !_rules[ruleopen]() { + goto l5 + } + if !_rules[ruleallargs]() { + goto l5 + } + { + position53, tokenIndex53 := position, tokenIndex + if !_rules[rulecomma]() { + goto l53 + } + goto l54 + l53: + position, tokenIndex = position53, tokenIndex53 + } + l54: + if !_rules[ruleclose]() { + goto l5 + } + { + add(ruleAction17, position) + } } l7: add(ruleCall, position6) @@ -1112,520 +1167,474 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position51, tokenIndex51 := position, tokenIndex + position56, tokenIndex56 := position, tokenIndex { - position52 := position + position57 := position { - position53, tokenIndex53 := position, tokenIndex + position58, tokenIndex58 := position, tokenIndex if !_rules[ruleCall]() { - goto l54 - } - l55: - { - position56, tokenIndex56 := position, tokenIndex - if !_rules[rulecomma]() { - goto l56 - } - if !_rules[ruleCall]() { - goto l56 - } - goto l55 - l56: - position, tokenIndex = position56, tokenIndex56 - } - { - position57, tokenIndex57 := position, tokenIndex - if !_rules[rulecomma]() { - goto l57 - } - if !_rules[ruleargs]() { - goto l57 - } - goto l58 - l57: - position, tokenIndex = position57, tokenIndex57 - } - l58: - goto l53 - l54: - position, tokenIndex = position53, tokenIndex53 - if !_rules[ruleargs]() { goto l59 } - goto l53 + l60: + { + position61, tokenIndex61 := position, tokenIndex + if !_rules[rulecomma]() { + goto l61 + } + if !_rules[ruleCall]() { + goto l61 + } + goto l60 + l61: + position, tokenIndex = position61, tokenIndex61 + } + { + position62, tokenIndex62 := position, tokenIndex + if !_rules[rulecomma]() { + goto l62 + } + if !_rules[ruleargs]() { + goto l62 + } + goto l63 + l62: + position, tokenIndex = position62, tokenIndex62 + } + l63: + goto l58 l59: - position, tokenIndex = position53, tokenIndex53 + position, tokenIndex = position58, tokenIndex58 + if !_rules[ruleargs]() { + goto l64 + } + goto l58 + l64: + position, tokenIndex = position58, tokenIndex58 if !_rules[rulesp]() { - goto l51 + goto l56 } } - l53: - add(ruleallargs, position52) + l58: + add(ruleallargs, position57) } return true - l51: - position, tokenIndex = position51, tokenIndex51 + l56: + position, tokenIndex = position56, tokenIndex56 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position60, tokenIndex60 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex { - position61 := position + position66 := position if !_rules[rulearg]() { - goto l60 + goto l65 } { - position62, tokenIndex62 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex if !_rules[rulecomma]() { - goto l62 + goto l67 } if !_rules[ruleargs]() { - goto l62 + goto l67 } - goto l63 - l62: - position, tokenIndex = position62, tokenIndex62 + goto l68 + l67: + position, tokenIndex = position67, tokenIndex67 } - l63: + l68: if !_rules[rulesp]() { - goto l60 + goto l65 } - add(ruleargs, position61) + add(ruleargs, position66) } return true - l60: - position, tokenIndex = position60, tokenIndex60 + l65: + position, tokenIndex = position65, tokenIndex65 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value) / conditional)> */ func() bool { - position64, tokenIndex64 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex { - position65 := position + position70 := position { - position66, tokenIndex66 := position, tokenIndex + position71, tokenIndex71 := position, tokenIndex if !_rules[rulefield]() { - goto l67 + goto l72 } if !_rules[rulesp]() { - goto l67 + goto l72 } if buffer[position] != rune('=') { - goto l67 + goto l72 } position++ if !_rules[rulesp]() { - goto l67 + goto l72 } if !_rules[rulevalue]() { - goto l67 + goto l72 } - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 + goto l71 + l72: + position, tokenIndex = position71, tokenIndex71 if !_rules[rulefield]() { - goto l68 + goto l73 } if !_rules[rulesp]() { - goto l68 + goto l73 } { - position69 := position + position74 := position { - position70, tokenIndex70 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex if buffer[position] != rune('>') { - goto l71 + goto l76 } position++ if buffer[position] != rune('<') { - goto l71 - } - position++ - { - add(ruleAction16, position) - } - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('<') { - goto l73 - } - position++ - if buffer[position] != rune('=') { - goto l73 - } - position++ - { - add(ruleAction17, position) - } - goto l70 - l73: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('>') { - goto l75 - } - position++ - if buffer[position] != rune('=') { - goto l75 + goto l76 } position++ { add(ruleAction18, position) } - goto l70 - l75: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('=') { - goto l77 + goto l75 + l76: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('<') { + goto l78 } position++ if buffer[position] != rune('=') { - goto l77 + goto l78 } position++ { add(ruleAction19, position) } - goto l70 - l77: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('!') { - goto l79 + goto l75 + l78: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('>') { + goto l80 } position++ if buffer[position] != rune('=') { - goto l79 + goto l80 } position++ { add(ruleAction20, position) } - goto l70 - l79: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('<') { - goto l81 + goto l75 + l80: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('=') { + goto l82 + } + position++ + if buffer[position] != rune('=') { + goto l82 } position++ { add(ruleAction21, position) } - goto l70 - l81: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('>') { - goto l68 + goto l75 + l82: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('!') { + goto l84 + } + position++ + if buffer[position] != rune('=') { + goto l84 } position++ { add(ruleAction22, position) } + goto l75 + l84: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('<') { + goto l86 + } + position++ + { + add(ruleAction23, position) + } + goto l75 + l86: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('>') { + goto l73 + } + position++ + { + add(ruleAction24, position) + } } - l70: - add(ruleCOND, position69) + l75: + add(ruleCOND, position74) } if !_rules[rulesp]() { - goto l68 + goto l73 } if !_rules[rulevalue]() { - goto l68 + goto l73 } - goto l66 - l68: - position, tokenIndex = position66, tokenIndex66 + goto l71 + l73: + position, tokenIndex = position71, tokenIndex71 { - position84 := position + position89 := position { - add(ruleAction23, position) + add(ruleAction25, position) } if !_rules[rulecondint]() { - goto l64 + goto l69 } if !_rules[rulecondLT]() { - goto l64 + goto l69 } { - position86 := position + position91 := position { - position87 := position + position92 := position if !_rules[rulefieldExpr]() { - goto l64 + goto l69 } - add(rulePegText, position87) + add(rulePegText, position92) } if !_rules[rulesp]() { - goto l64 + goto l69 } { - add(ruleAction27, position) + add(ruleAction29, position) } - add(rulecondfield, position86) + add(rulecondfield, position91) } if !_rules[rulecondLT]() { - goto l64 + goto l69 } if !_rules[rulecondint]() { - goto l64 + goto l69 } { - add(ruleAction24, position) + add(ruleAction26, position) } - add(ruleconditional, position84) + add(ruleconditional, position89) } } - l66: - add(rulearg, position65) + l71: + add(rulearg, position70) } return true - l64: - position, tokenIndex = position64, tokenIndex64 + l69: + position, tokenIndex = position69, tokenIndex69 return false }, - /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ + /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ nil, - /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ + /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ func() bool { - position92, tokenIndex92 := position, tokenIndex + position97, tokenIndex97 := position, tokenIndex { - position93 := position + position98 := position { - position94 := position + position99 := position { - position95, tokenIndex95 := position, tokenIndex + position100, tokenIndex100 := position, tokenIndex { - position97, tokenIndex97 := position, tokenIndex + position102, tokenIndex102 := position, tokenIndex if buffer[position] != rune('-') { - goto l97 + goto l102 } position++ - goto l98 - l97: - position, tokenIndex = position97, tokenIndex97 + goto l103 + l102: + position, tokenIndex = position102, tokenIndex102 } - l98: + l103: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l96 + goto l101 } position++ - l99: + l104: { - position100, tokenIndex100 := position, tokenIndex + position105, tokenIndex105 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l100 + goto l105 } position++ - goto l99 - l100: - position, tokenIndex = position100, tokenIndex100 + goto l104 + l105: + position, tokenIndex = position105, tokenIndex105 } - goto l95 - l96: - position, tokenIndex = position95, tokenIndex95 + goto l100 + l101: + position, tokenIndex = position100, tokenIndex100 if buffer[position] != rune('0') { - goto l92 + goto l97 } position++ } - l95: - add(rulePegText, position94) + l100: + add(rulePegText, position99) } if !_rules[rulesp]() { - goto l92 + goto l97 } { - add(ruleAction25, position) + add(ruleAction27, position) } - add(rulecondint, position93) + add(rulecondint, position98) } return true - l92: - position, tokenIndex = position92, tokenIndex92 + l97: + position, tokenIndex = position97, tokenIndex97 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position102, tokenIndex102 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex { - position103 := position + position108 := position { - position104 := position + position109 := position { - position105, tokenIndex105 := position, tokenIndex + position110, tokenIndex110 := position, tokenIndex if buffer[position] != rune('<') { - goto l106 + goto l111 } position++ if buffer[position] != rune('=') { - goto l106 + goto l111 } position++ - goto l105 - l106: - position, tokenIndex = position105, tokenIndex105 + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 if buffer[position] != rune('<') { - goto l102 + goto l107 } position++ } - l105: - add(rulePegText, position104) + l110: + add(rulePegText, position109) } if !_rules[rulesp]() { - goto l102 + goto l107 } { - add(ruleAction26, position) + add(ruleAction28, position) } - add(rulecondLT, position103) + add(rulecondLT, position108) } return true - l102: - position, tokenIndex = position102, tokenIndex102 + l107: + position, tokenIndex = position107, tokenIndex107 return false }, - /* 9 condfield <- <( sp Action27)> */ + /* 9 condfield <- <( sp Action29)> */ nil, - /* 10 value <- <(item / (lbrack Action28 list rbrack Action29))> */ + /* 10 value <- <(item / (lbrack Action30 list rbrack Action31))> */ func() bool { - position109, tokenIndex109 := position, tokenIndex + position114, tokenIndex114 := position, tokenIndex { - position110 := position + position115 := position { - position111, tokenIndex111 := position, tokenIndex + position116, tokenIndex116 := position, tokenIndex if !_rules[ruleitem]() { - goto l112 + goto l117 } - goto l111 - l112: - position, tokenIndex = position111, tokenIndex111 + goto l116 + l117: + position, tokenIndex = position116, tokenIndex116 { - position113 := position + position118 := position if buffer[position] != rune('[') { - goto l109 + goto l114 } position++ if !_rules[rulesp]() { - goto l109 + goto l114 } - add(rulelbrack, position113) - } - { - add(ruleAction28, position) - } - if !_rules[rulelist]() { - goto l109 - } - { - position115 := position - if !_rules[rulesp]() { - goto l109 - } - if buffer[position] != rune(']') { - goto l109 - } - position++ - if !_rules[rulesp]() { - goto l109 - } - add(rulerbrack, position115) - } - { - add(ruleAction29, position) - } - } - l111: - add(rulevalue, position110) - } - return true - l109: - position, tokenIndex = position109, tokenIndex109 - return false - }, - /* 11 list <- <(item (comma list)?)> */ - func() bool { - position117, tokenIndex117 := position, tokenIndex - { - position118 := position - if !_rules[ruleitem]() { - goto l117 - } - { - position119, tokenIndex119 := position, tokenIndex - if !_rules[rulecomma]() { - goto l119 - } - if !_rules[rulelist]() { - goto l119 - } - goto l120 - l119: - position, tokenIndex = position119, tokenIndex119 - } - l120: - add(rulelist, position118) - } - return true - l117: - position, tokenIndex = position117, tokenIndex117 - return false - }, - /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (timestampfmt Action33) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action34) / (<('-'? '.' [0-9]+)> Action35) / ( Action36 open allargs comma? close Action37) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action38) / (<('"' doublequotedstring '"')> Action39) / ('\'' '\'' Action40))> */ - func() bool { - position121, tokenIndex121 := position, tokenIndex - { - position122 := position - { - position123, tokenIndex123 := position, tokenIndex - if buffer[position] != rune('n') { - goto l124 - } - position++ - if buffer[position] != rune('u') { - goto l124 - } - position++ - if buffer[position] != rune('l') { - goto l124 - } - position++ - if buffer[position] != rune('l') { - goto l124 - } - position++ - { - position125, tokenIndex125 := position, tokenIndex - { - position126, tokenIndex126 := position, tokenIndex - if !_rules[rulecomma]() { - goto l127 - } - goto l126 - l127: - position, tokenIndex = position126, tokenIndex126 - if !_rules[rulesp]() { - goto l124 - } - if !_rules[ruleclose]() { - goto l124 - } - } - l126: - position, tokenIndex = position125, tokenIndex125 + add(rulelbrack, position118) } { add(ruleAction30, position) } - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('t') { - goto l129 + if !_rules[rulelist]() { + goto l114 } - position++ - if buffer[position] != rune('r') { + { + position120 := position + if !_rules[rulesp]() { + goto l114 + } + if buffer[position] != rune(']') { + goto l114 + } + position++ + if !_rules[rulesp]() { + goto l114 + } + add(rulerbrack, position120) + } + { + add(ruleAction31, position) + } + } + l116: + add(rulevalue, position115) + } + return true + l114: + position, tokenIndex = position114, tokenIndex114 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position122, tokenIndex122 := position, tokenIndex + { + position123 := position + if !_rules[ruleitem]() { + goto l122 + } + { + position124, tokenIndex124 := position, tokenIndex + if !_rules[rulecomma]() { + goto l124 + } + if !_rules[rulelist]() { + goto l124 + } + goto l125 + l124: + position, tokenIndex = position124, tokenIndex124 + } + l125: + add(rulelist, position123) + } + return true + l122: + position, tokenIndex = position122, tokenIndex122 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (timestampfmt Action35) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action36) / (<('-'? '.' [0-9]+)> Action37) / ( Action38 open allargs comma? close Action39) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action40) / (<('"' doublequotedstring '"')> Action41) / ('\'' '\'' Action42))> */ + func() bool { + position126, tokenIndex126 := position, tokenIndex + { + position127 := position + { + position128, tokenIndex128 := position, tokenIndex + if buffer[position] != rune('n') { goto l129 } position++ @@ -1633,7 +1642,11 @@ func (p *PQL) Init() { goto l129 } position++ - if buffer[position] != rune('e') { + if buffer[position] != rune('l') { + goto l129 + } + position++ + if buffer[position] != rune('l') { goto l129 } position++ @@ -1658,24 +1671,20 @@ func (p *PQL) Init() { position, tokenIndex = position130, tokenIndex130 } { - add(ruleAction31, position) + add(ruleAction32, position) } - goto l123 + goto l128 l129: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('f') { + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('t') { goto l134 } position++ - if buffer[position] != rune('a') { + if buffer[position] != rune('r') { goto l134 } position++ - if buffer[position] != rune('l') { - goto l134 - } - position++ - if buffer[position] != rune('s') { + if buffer[position] != rune('u') { goto l134 } position++ @@ -1703,898 +1712,944 @@ func (p *PQL) Init() { l136: position, tokenIndex = position135, tokenIndex135 } - { - add(ruleAction32, position) - } - goto l123 - l134: - position, tokenIndex = position123, tokenIndex123 - if !_rules[ruletimestampfmt]() { - goto l139 - } { add(ruleAction33, position) } - goto l123 - l139: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l134: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('f') { + goto l139 + } + position++ + if buffer[position] != rune('a') { + goto l139 + } + position++ + if buffer[position] != rune('l') { + goto l139 + } + position++ + if buffer[position] != rune('s') { + goto l139 + } + position++ + if buffer[position] != rune('e') { + goto l139 + } + position++ { - position142 := position + position140, tokenIndex140 := position, tokenIndex { - position143, tokenIndex143 := position, tokenIndex - if buffer[position] != rune('-') { - goto l143 + position141, tokenIndex141 := position, tokenIndex + if !_rules[rulecomma]() { + goto l142 } - position++ - goto l144 - l143: - position, tokenIndex = position143, tokenIndex143 - } - l144: - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l141 - } - position++ - l145: - { - position146, tokenIndex146 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l146 + l142: + position, tokenIndex = position141, tokenIndex141 + if !_rules[rulesp]() { + goto l139 } - position++ - goto l145 - l146: - position, tokenIndex = position146, tokenIndex146 - } - { - position147, tokenIndex147 := position, tokenIndex - if buffer[position] != rune('.') { - goto l147 + if !_rules[ruleclose]() { + goto l139 } - position++ - l149: - { - position150, tokenIndex150 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l150 - } - position++ - goto l149 - l150: - position, tokenIndex = position150, tokenIndex150 - } - goto l148 - l147: - position, tokenIndex = position147, tokenIndex147 } - l148: - add(rulePegText, position142) + l141: + position, tokenIndex = position140, tokenIndex140 } { add(ruleAction34, position) } - goto l123 - l141: - position, tokenIndex = position123, tokenIndex123 - { - position153 := position - { - position154, tokenIndex154 := position, tokenIndex - if buffer[position] != rune('-') { - goto l154 - } - position++ - goto l155 - l154: - position, tokenIndex = position154, tokenIndex154 - } - l155: - if buffer[position] != rune('.') { - goto l152 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 - } - position++ - l156: - { - position157, tokenIndex157 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l157 - } - position++ - goto l156 - l157: - position, tokenIndex = position157, tokenIndex157 - } - add(rulePegText, position153) + goto l128 + l139: + position, tokenIndex = position128, tokenIndex128 + if !_rules[ruletimestampfmt]() { + goto l144 } { add(ruleAction35, position) } - goto l123 - l152: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l144: + position, tokenIndex = position128, tokenIndex128 { - position160 := position - if !_rules[ruleIDENT]() { - goto l159 + position147 := position + { + position148, tokenIndex148 := position, tokenIndex + if buffer[position] != rune('-') { + goto l148 + } + position++ + goto l149 + l148: + position, tokenIndex = position148, tokenIndex148 } - add(rulePegText, position160) + l149: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l146 + } + position++ + l150: + { + position151, tokenIndex151 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l151 + } + position++ + goto l150 + l151: + position, tokenIndex = position151, tokenIndex151 + } + { + position152, tokenIndex152 := position, tokenIndex + if buffer[position] != rune('.') { + goto l152 + } + position++ + l154: + { + position155, tokenIndex155 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l155 + } + position++ + goto l154 + l155: + position, tokenIndex = position155, tokenIndex155 + } + goto l153 + l152: + position, tokenIndex = position152, tokenIndex152 + } + l153: + add(rulePegText, position147) } { add(ruleAction36, position) } - if !_rules[ruleopen]() { - goto l159 - } - if !_rules[ruleallargs]() { - goto l159 - } + goto l128 + l146: + position, tokenIndex = position128, tokenIndex128 { - position162, tokenIndex162 := position, tokenIndex - if !_rules[rulecomma]() { - goto l162 + position158 := position + { + position159, tokenIndex159 := position, tokenIndex + if buffer[position] != rune('-') { + goto l159 + } + position++ + goto l160 + l159: + position, tokenIndex = position159, tokenIndex159 } - goto l163 - l162: - position, tokenIndex = position162, tokenIndex162 - } - l163: - if !_rules[ruleclose]() { - goto l159 + l160: + if buffer[position] != rune('.') { + goto l157 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l157 + } + position++ + l161: + { + position162, tokenIndex162 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l162 + } + position++ + goto l161 + l162: + position, tokenIndex = position162, tokenIndex162 + } + add(rulePegText, position158) } { add(ruleAction37, position) } - goto l123 - l159: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l157: + position, tokenIndex = position128, tokenIndex128 { - position166 := position - { - position169, tokenIndex169 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l170 - } - position++ - goto l169 - l170: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l171 - } - position++ - goto l169 - l171: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l172 - } - position++ - goto l169 - l172: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune('-') { - goto l173 - } - position++ - goto l169 - l173: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune('_') { - goto l174 - } - position++ - goto l169 - l174: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune(':') { - goto l165 - } - position++ + position165 := position + if !_rules[ruleIDENT]() { + goto l164 } - l169: - l167: - { - position168, tokenIndex168 := position, tokenIndex - { - position175, tokenIndex175 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l176 - } - position++ - goto l175 - l176: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l177 - } - position++ - goto l175 - l177: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l178 - } - position++ - goto l175 - l178: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('-') { - goto l179 - } - position++ - goto l175 - l179: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('_') { - goto l180 - } - position++ - goto l175 - l180: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune(':') { - goto l168 - } - position++ - } - l175: - goto l167 - l168: - position, tokenIndex = position168, tokenIndex168 - } - add(rulePegText, position166) + add(rulePegText, position165) } { add(ruleAction38, position) } - goto l123 - l165: - position, tokenIndex = position123, tokenIndex123 + if !_rules[ruleopen]() { + goto l164 + } + if !_rules[ruleallargs]() { + goto l164 + } { - position183 := position - if buffer[position] != rune('"') { - goto l182 + position167, tokenIndex167 := position, tokenIndex + if !_rules[rulecomma]() { + goto l167 } - position++ - if !_rules[ruledoublequotedstring]() { - goto l182 - } - if buffer[position] != rune('"') { - goto l182 - } - position++ - add(rulePegText, position183) + goto l168 + l167: + position, tokenIndex = position167, tokenIndex167 + } + l168: + if !_rules[ruleclose]() { + goto l164 } { add(ruleAction39, position) } - goto l123 - l182: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('\'') { - goto l121 - } - position++ + goto l128 + l164: + position, tokenIndex = position128, tokenIndex128 { - position185 := position - if !_rules[rulesinglequotedstring]() { - goto l121 + position171 := position + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l175 + } + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l176 + } + position++ + goto l174 + l176: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l177 + } + position++ + goto l174 + l177: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('-') { + goto l178 + } + position++ + goto l174 + l178: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('_') { + goto l179 + } + position++ + goto l174 + l179: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune(':') { + goto l170 + } + position++ } - add(rulePegText, position185) + l174: + l172: + { + position173, tokenIndex173 := position, tokenIndex + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l173 + } + position++ + } + l180: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + add(rulePegText, position171) } - if buffer[position] != rune('\'') { - goto l121 - } - position++ { add(ruleAction40, position) } + goto l128 + l170: + position, tokenIndex = position128, tokenIndex128 + { + position188 := position + if buffer[position] != rune('"') { + goto l187 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l187 + } + if buffer[position] != rune('"') { + goto l187 + } + position++ + add(rulePegText, position188) + } + { + add(ruleAction41, position) + } + goto l128 + l187: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('\'') { + goto l126 + } + position++ + { + position190 := position + if !_rules[rulesinglequotedstring]() { + goto l126 + } + add(rulePegText, position190) + } + if buffer[position] != rune('\'') { + goto l126 + } + position++ + { + add(ruleAction42, position) + } } - l123: - add(ruleitem, position122) + l128: + add(ruleitem, position127) } return true - l121: - position, tokenIndex = position121, tokenIndex121 + l126: + position, tokenIndex = position126, tokenIndex126 return false }, /* 13 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position188 := position - l189: + position193 := position + l194: { - position190, tokenIndex190 := position, tokenIndex + position195, tokenIndex195 := position, tokenIndex { - position191, tokenIndex191 := position, tokenIndex + position196, tokenIndex196 := position, tokenIndex if buffer[position] != rune('\\') { - goto l192 + goto l197 } position++ if buffer[position] != rune('"') { - goto l192 + goto l197 } position++ - goto l191 - l192: - position, tokenIndex = position191, tokenIndex191 + goto l196 + l197: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l193 + goto l198 } position++ if buffer[position] != rune('\\') { - goto l193 + goto l198 } position++ - goto l191 - l193: - position, tokenIndex = position191, tokenIndex191 + goto l196 + l198: + position, tokenIndex = position196, tokenIndex196 { - position194, tokenIndex194 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex if buffer[position] != rune('"') { - goto l194 + goto l199 } position++ - goto l190 - l194: - position, tokenIndex = position194, tokenIndex194 + goto l195 + l199: + position, tokenIndex = position199, tokenIndex199 } if !matchDot() { - goto l190 + goto l195 } } - l191: - goto l189 - l190: - position, tokenIndex = position190, tokenIndex190 + l196: + goto l194 + l195: + position, tokenIndex = position195, tokenIndex195 } - add(ruledoublequotedstring, position188) + add(ruledoublequotedstring, position193) } return true }, /* 14 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position196 := position - l197: + position201 := position + l202: { - position198, tokenIndex198 := position, tokenIndex + position203, tokenIndex203 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position204, tokenIndex204 := position, tokenIndex if buffer[position] != rune('\\') { - goto l200 + goto l205 } position++ if buffer[position] != rune('\'') { - goto l200 + goto l205 } position++ - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l204 + l205: + position, tokenIndex = position204, tokenIndex204 if buffer[position] != rune('\\') { - goto l201 + goto l206 } position++ if buffer[position] != rune('\\') { - goto l201 + goto l206 } position++ - goto l199 - l201: - position, tokenIndex = position199, tokenIndex199 + goto l204 + l206: + position, tokenIndex = position204, tokenIndex204 { - position202, tokenIndex202 := position, tokenIndex + position207, tokenIndex207 := position, tokenIndex if buffer[position] != rune('\'') { - goto l202 + goto l207 } position++ - goto l198 - l202: - position, tokenIndex = position202, tokenIndex202 + goto l203 + l207: + position, tokenIndex = position207, tokenIndex207 } if !matchDot() { - goto l198 + goto l203 } } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l204: + goto l202 + l203: + position, tokenIndex = position203, tokenIndex203 } - add(rulesinglequotedstring, position196) + add(rulesinglequotedstring, position201) } return true }, /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position203, tokenIndex203 := position, tokenIndex + position208, tokenIndex208 := position, tokenIndex { - position204 := position + position209 := position { - position205, tokenIndex205 := position, tokenIndex + position210, tokenIndex210 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l206 + goto l211 } position++ - goto l205 - l206: - position, tokenIndex = position205, tokenIndex205 + goto l210 + l211: + position, tokenIndex = position210, tokenIndex210 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l203 + goto l208 } position++ } - l205: - l207: + l210: + l212: { - position208, tokenIndex208 := position, tokenIndex + position213, tokenIndex213 := position, tokenIndex { - position209, tokenIndex209 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l210 + goto l215 } position++ - goto l209 - l210: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l215: + position, tokenIndex = position214, tokenIndex214 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l211 + goto l216 } position++ - goto l209 - l211: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l216: + position, tokenIndex = position214, tokenIndex214 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l212 + goto l217 } position++ - goto l209 - l212: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l217: + position, tokenIndex = position214, tokenIndex214 if buffer[position] != rune('_') { + goto l218 + } + position++ + goto l214 + l218: + position, tokenIndex = position214, tokenIndex214 + if buffer[position] != rune('-') { goto l213 } position++ - goto l209 - l213: - position, tokenIndex = position209, tokenIndex209 - if buffer[position] != rune('-') { - goto l208 - } - position++ } - l209: - goto l207 - l208: - position, tokenIndex = position208, tokenIndex208 + l214: + goto l212 + l213: + position, tokenIndex = position213, tokenIndex213 } - add(rulefieldExpr, position204) + add(rulefieldExpr, position209) } return true - l203: - position, tokenIndex = position203, tokenIndex203 + l208: + position, tokenIndex = position208, tokenIndex208 return false }, - /* 16 field <- <(<(fieldExpr / reserved)> Action41)> */ + /* 16 field <- <(<(fieldExpr / reserved)> Action43)> */ func() bool { - position214, tokenIndex214 := position, tokenIndex + position219, tokenIndex219 := position, tokenIndex { - position215 := position + position220 := position { - position216 := position + position221 := position { - position217, tokenIndex217 := position, tokenIndex + position222, tokenIndex222 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l218 + goto l223 } - goto l217 - l218: - position, tokenIndex = position217, tokenIndex217 + goto l222 + l223: + position, tokenIndex = position222, tokenIndex222 { - position219 := position + position224 := position { - position220, tokenIndex220 := position, tokenIndex + position225, tokenIndex225 := position, tokenIndex if buffer[position] != rune('_') { - goto l221 + goto l226 } position++ if buffer[position] != rune('r') { - goto l221 + goto l226 } position++ if buffer[position] != rune('o') { - goto l221 + goto l226 } position++ if buffer[position] != rune('w') { - goto l221 + goto l226 } position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l226: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l222 + goto l227 } position++ if buffer[position] != rune('c') { - goto l222 + goto l227 } position++ if buffer[position] != rune('o') { - goto l222 + goto l227 } position++ if buffer[position] != rune('l') { - goto l222 + goto l227 } position++ - goto l220 - l222: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l227: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l223 + goto l228 } position++ if buffer[position] != rune('s') { - goto l223 + goto l228 } position++ if buffer[position] != rune('t') { - goto l223 + goto l228 } position++ if buffer[position] != rune('a') { - goto l223 + goto l228 } position++ if buffer[position] != rune('r') { - goto l223 + goto l228 } position++ if buffer[position] != rune('t') { - goto l223 + goto l228 } position++ - goto l220 - l223: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l228: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l224 + goto l229 } position++ if buffer[position] != rune('e') { - goto l224 + goto l229 } position++ if buffer[position] != rune('n') { - goto l224 + goto l229 } position++ if buffer[position] != rune('d') { - goto l224 + goto l229 } position++ - goto l220 - l224: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l229: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l225 + goto l230 } position++ if buffer[position] != rune('t') { - goto l225 + goto l230 } position++ if buffer[position] != rune('i') { - goto l225 + goto l230 } position++ if buffer[position] != rune('m') { - goto l225 + goto l230 } position++ if buffer[position] != rune('e') { - goto l225 + goto l230 } position++ if buffer[position] != rune('s') { - goto l225 + goto l230 } position++ if buffer[position] != rune('t') { - goto l225 + goto l230 } position++ if buffer[position] != rune('a') { - goto l225 + goto l230 } position++ if buffer[position] != rune('m') { - goto l225 + goto l230 } position++ if buffer[position] != rune('p') { - goto l225 + goto l230 } position++ - goto l220 - l225: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l230: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l214 + goto l219 } position++ if buffer[position] != rune('f') { - goto l214 + goto l219 } position++ if buffer[position] != rune('i') { - goto l214 + goto l219 } position++ if buffer[position] != rune('e') { - goto l214 + goto l219 } position++ if buffer[position] != rune('l') { - goto l214 + goto l219 } position++ if buffer[position] != rune('d') { - goto l214 + goto l219 } position++ } - l220: - add(rulereserved, position219) + l225: + add(rulereserved, position224) } } - l217: - add(rulePegText, position216) + l222: + add(rulePegText, position221) } { - add(ruleAction41, position) + add(ruleAction43, position) } - add(rulefield, position215) + add(rulefield, position220) } return true - l214: - position, tokenIndex = position214, tokenIndex214 + l219: + position, tokenIndex = position219, tokenIndex219 return false }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <( Action42)> */ + /* 18 posfield <- <( Action44)> */ func() bool { - position228, tokenIndex228 := position, tokenIndex + position233, tokenIndex233 := position, tokenIndex { - position229 := position + position234 := position { - position230 := position + position235 := position if !_rules[rulefieldExpr]() { - goto l228 + goto l233 } - add(rulePegText, position230) + add(rulePegText, position235) } { - add(ruleAction42, position) + add(ruleAction44, position) } - add(ruleposfield, position229) + add(ruleposfield, position234) } return true - l228: - position, tokenIndex = position228, tokenIndex228 + l233: + position, tokenIndex = position233, tokenIndex233 return false }, /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position232, tokenIndex232 := position, tokenIndex + position237, tokenIndex237 := position, tokenIndex { - position233 := position + position238 := position { - position234, tokenIndex234 := position, tokenIndex + position239, tokenIndex239 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l235 + goto l240 } position++ - l236: + l241: { - position237, tokenIndex237 := position, tokenIndex + position242, tokenIndex242 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l237 + goto l242 } position++ - goto l236 - l237: - position, tokenIndex = position237, tokenIndex237 + goto l241 + l242: + position, tokenIndex = position242, tokenIndex242 } - goto l234 - l235: - position, tokenIndex = position234, tokenIndex234 + goto l239 + l240: + position, tokenIndex = position239, tokenIndex239 if buffer[position] != rune('0') { - goto l232 + goto l237 } position++ } - l234: - add(ruleuint, position233) + l239: + add(ruleuint, position238) } return true - l232: - position, tokenIndex = position232, tokenIndex232 + l237: + position, tokenIndex = position237, tokenIndex237 return false }, - /* 20 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ + /* 20 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ func() bool { - position238, tokenIndex238 := position, tokenIndex + position243, tokenIndex243 := position, tokenIndex { - position239 := position + position244 := position { - position240, tokenIndex240 := position, tokenIndex - { - position242 := position - if !_rules[ruleuint]() { - goto l241 - } - add(rulePegText, position242) - } - { - add(ruleAction43, position) - } - goto l240 - l241: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('\'') { - goto l244 - } - position++ - { - position245 := position - if !_rules[rulesinglequotedstring]() { - goto l244 - } - add(rulePegText, position245) - } - if buffer[position] != rune('\'') { - goto l244 - } - position++ - { - add(ruleAction44, position) - } - goto l240 - l244: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('"') { - goto l238 - } - position++ + position245, tokenIndex245 := position, tokenIndex { position247 := position - if !_rules[ruledoublequotedstring]() { - goto l238 + if !_rules[ruleuint]() { + goto l246 } add(rulePegText, position247) } - if buffer[position] != rune('"') { - goto l238 - } - position++ { add(ruleAction45, position) } + goto l245 + l246: + position, tokenIndex = position245, tokenIndex245 + if buffer[position] != rune('\'') { + goto l249 + } + position++ + { + position250 := position + if !_rules[rulesinglequotedstring]() { + goto l249 + } + add(rulePegText, position250) + } + if buffer[position] != rune('\'') { + goto l249 + } + position++ + { + add(ruleAction46, position) + } + goto l245 + l249: + position, tokenIndex = position245, tokenIndex245 + if buffer[position] != rune('"') { + goto l243 + } + position++ + { + position252 := position + if !_rules[ruledoublequotedstring]() { + goto l243 + } + add(rulePegText, position252) + } + if buffer[position] != rune('"') { + goto l243 + } + position++ + { + add(ruleAction47, position) + } } - l240: - add(rulecol, position239) + l245: + add(rulecol, position244) } return true - l238: - position, tokenIndex = position238, tokenIndex238 + l243: + position, tokenIndex = position243, tokenIndex243 return false }, - /* 21 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ + /* 21 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ nil, /* 22 open <- <('(' sp)> */ func() bool { - position250, tokenIndex250 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex { - position251 := position + position256 := position if buffer[position] != rune('(') { - goto l250 + goto l255 } position++ if !_rules[rulesp]() { - goto l250 + goto l255 } - add(ruleopen, position251) + add(ruleopen, position256) } return true - l250: - position, tokenIndex = position250, tokenIndex250 + l255: + position, tokenIndex = position255, tokenIndex255 return false }, /* 23 close <- <(')' sp)> */ func() bool { - position252, tokenIndex252 := position, tokenIndex + position257, tokenIndex257 := position, tokenIndex { - position253 := position + position258 := position if buffer[position] != rune(')') { - goto l252 + goto l257 } position++ if !_rules[rulesp]() { - goto l252 + goto l257 } - add(ruleclose, position253) + add(ruleclose, position258) } return true - l252: - position, tokenIndex = position252, tokenIndex252 + l257: + position, tokenIndex = position257, tokenIndex257 return false }, /* 24 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position255 := position - l256: + position260 := position + l261: { - position257, tokenIndex257 := position, tokenIndex + position262, tokenIndex262 := position, tokenIndex { - position258, tokenIndex258 := position, tokenIndex + position263, tokenIndex263 := position, tokenIndex if buffer[position] != rune(' ') { - goto l259 + goto l264 } position++ - goto l258 - l259: - position, tokenIndex = position258, tokenIndex258 + goto l263 + l264: + position, tokenIndex = position263, tokenIndex263 if buffer[position] != rune('\t') { - goto l260 + goto l265 } position++ - goto l258 - l260: - position, tokenIndex = position258, tokenIndex258 + goto l263 + l265: + position, tokenIndex = position263, tokenIndex263 if buffer[position] != rune('\n') { - goto l257 + goto l262 } position++ } - l258: - goto l256 - l257: - position, tokenIndex = position257, tokenIndex257 + l263: + goto l261 + l262: + position, tokenIndex = position262, tokenIndex262 } - add(rulesp, position255) + add(rulesp, position260) } return true }, /* 25 comma <- <(sp ',' sp)> */ func() bool { - position261, tokenIndex261 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position262 := position + position267 := position if !_rules[rulesp]() { - goto l261 + goto l266 } if buffer[position] != rune(',') { - goto l261 + goto l266 } position++ if !_rules[rulesp]() { - goto l261 + goto l266 } - add(rulecomma, position262) + add(rulecomma, position267) } return true - l261: - position, tokenIndex = position261, tokenIndex261 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 26 lbrack <- <('[' sp)> */ @@ -2603,207 +2658,207 @@ func (p *PQL) Init() { nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position265, tokenIndex265 := position, tokenIndex + position270, tokenIndex270 := position, tokenIndex { - position266 := position + position271 := position { - position267, tokenIndex267 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l268 + goto l273 } position++ - goto l267 - l268: - position, tokenIndex = position267, tokenIndex267 + goto l272 + l273: + position, tokenIndex = position272, tokenIndex272 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l265 + goto l270 } position++ } - l267: - l269: + l272: + l274: { - position270, tokenIndex270 := position, tokenIndex + position275, tokenIndex275 := position, tokenIndex { - position271, tokenIndex271 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l272 + goto l277 } position++ - goto l271 - l272: - position, tokenIndex = position271, tokenIndex271 + goto l276 + l277: + position, tokenIndex = position276, tokenIndex276 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l273 + goto l278 } position++ - goto l271 - l273: - position, tokenIndex = position271, tokenIndex271 + goto l276 + l278: + position, tokenIndex = position276, tokenIndex276 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l270 + goto l275 } position++ } - l271: - goto l269 - l270: - position, tokenIndex = position270, tokenIndex270 + l276: + goto l274 + l275: + position, tokenIndex = position275, tokenIndex275 } - add(ruleIDENT, position266) + add(ruleIDENT, position271) } return true - l265: - position, tokenIndex = position265, tokenIndex265 + l270: + position, tokenIndex = position270, tokenIndex270 return false }, /* 29 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position274, tokenIndex274 := position, tokenIndex + position279, tokenIndex279 := position, tokenIndex { - position275 := position + position280 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('-') { - goto l274 + goto l279 } position++ { - position276, tokenIndex276 := position, tokenIndex + position281, tokenIndex281 := position, tokenIndex if buffer[position] != rune('0') { - goto l277 + goto l282 } position++ - goto l276 - l277: - position, tokenIndex = position276, tokenIndex276 + goto l281 + l282: + position, tokenIndex = position281, tokenIndex281 if buffer[position] != rune('1') { - goto l274 + goto l279 } position++ } - l276: + l281: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('-') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('T') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune(':') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ - add(ruletimestampbasicfmt, position275) + add(ruletimestampbasicfmt, position280) } return true - l274: - position, tokenIndex = position274, tokenIndex274 + l279: + position, tokenIndex = position279, tokenIndex279 return false }, /* 30 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position278, tokenIndex278 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position279 := position + position284 := position { - position280, tokenIndex280 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex if buffer[position] != rune('"') { - goto l281 + goto l286 } position++ { - position282 := position + position287 := position if !_rules[ruletimestampbasicfmt]() { - goto l281 + goto l286 } - add(rulePegText, position282) + add(rulePegText, position287) } if buffer[position] != rune('"') { - goto l281 + goto l286 } position++ - goto l280 - l281: - position, tokenIndex = position280, tokenIndex280 + goto l285 + l286: + position, tokenIndex = position285, tokenIndex285 if buffer[position] != rune('\'') { - goto l283 + goto l288 } position++ { - position284 := position + position289 := position + if !_rules[ruletimestampbasicfmt]() { + goto l288 + } + add(rulePegText, position289) + } + if buffer[position] != rune('\'') { + goto l288 + } + position++ + goto l285 + l288: + position, tokenIndex = position285, tokenIndex285 + { + position290 := position if !_rules[ruletimestampbasicfmt]() { goto l283 } - add(rulePegText, position284) - } - if buffer[position] != rune('\'') { - goto l283 - } - position++ - goto l280 - l283: - position, tokenIndex = position280, tokenIndex280 - { - position285 := position - if !_rules[ruletimestampbasicfmt]() { - goto l278 - } - add(rulePegText, position285) + add(rulePegText, position290) } } - l280: - add(ruletimestampfmt, position279) + l285: + add(ruletimestampfmt, position284) } return true - l278: - position, tokenIndex = position278, tokenIndex278 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, - /* 31 timestamp <- <( Action49)> */ + /* 31 timestamp <- <( Action51)> */ nil, /* 33 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2833,78 +2888,82 @@ func (p *PQL) Init() { nil, /* 46 Action13 <- <{p.endCall()}> */ nil, + /* 47 Action14 <- <{p.startCall("Rows")}> */ nil, - /* 48 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action15 <- <{p.endCall()}> */ nil, - /* 49 Action15 <- <{ p.endCall() }> */ nil, - /* 50 Action16 <- <{ p.addBTWN() }> */ + /* 50 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 51 Action17 <- <{ p.addLTE() }> */ + /* 51 Action17 <- <{ p.endCall() }> */ nil, - /* 52 Action18 <- <{ p.addGTE() }> */ + /* 52 Action18 <- <{ p.addBTWN() }> */ nil, - /* 53 Action19 <- <{ p.addEQ() }> */ + /* 53 Action19 <- <{ p.addLTE() }> */ nil, - /* 54 Action20 <- <{ p.addNEQ() }> */ + /* 54 Action20 <- <{ p.addGTE() }> */ nil, - /* 55 Action21 <- <{ p.addLT() }> */ + /* 55 Action21 <- <{ p.addEQ() }> */ nil, - /* 56 Action22 <- <{ p.addGT() }> */ + /* 56 Action22 <- <{ p.addNEQ() }> */ nil, - /* 57 Action23 <- <{p.startConditional()}> */ + /* 57 Action23 <- <{ p.addLT() }> */ nil, - /* 58 Action24 <- <{p.endConditional()}> */ + /* 58 Action24 <- <{ p.addGT() }> */ nil, - /* 59 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action25 <- <{p.startConditional()}> */ nil, - /* 60 Action26 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action26 <- <{p.endConditional()}> */ nil, /* 61 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 62 Action28 <- <{ p.startList() }> */ + /* 62 Action28 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 63 Action29 <- <{ p.endList() }> */ + /* 63 Action29 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action30 <- <{ p.addVal(nil) }> */ + /* 64 Action30 <- <{ p.startList() }> */ nil, - /* 65 Action31 <- <{ p.addVal(true) }> */ + /* 65 Action31 <- <{ p.endList() }> */ nil, - /* 66 Action32 <- <{ p.addVal(false) }> */ + /* 66 Action32 <- <{ p.addVal(nil) }> */ nil, - /* 67 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 67 Action33 <- <{ p.addVal(true) }> */ nil, - /* 68 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 68 Action34 <- <{ p.addVal(false) }> */ nil, - /* 69 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 70 Action36 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 70 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 71 Action37 <- <{ p.addVal(p.endCall()) }> */ + /* 71 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 72 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action38 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 73 Action39 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 73 Action39 <- <{ p.addVal(p.endCall()) }> */ nil, /* 74 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 75 Action41 <- <{ p.addField(buffer[begin:end]) }> */ + /* 75 Action41 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 76 Action42 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 76 Action42 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action43 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 78 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action44 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 79 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 80 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 80 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 81 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 81 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 82 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 82 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 83 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 83 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 84 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 85 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/roaring/roaring.go b/roaring/roaring.go index 7624b9f13..a0568dbc2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -163,9 +163,8 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { } // Apply to the in-memory bitmap. - if op.apply(b) { + if b.DirectAdd(v) { changed = true - } } @@ -233,6 +232,18 @@ func (b *Bitmap) Count() (n uint64) { return b.Containers.Count() } +// Size returns the number of bytes required for the bitmap. +func (b *Bitmap) Size() int { + numbytes := 0 + citer, _ := b.Containers.Iterator(0) + for citer.Next() { + _, c := citer.Value() + numbytes += c.size() + + } + return numbytes +} + // CountRange returns the number of bits set between [start, end). func (b *Bitmap) CountRange(start, end uint64) (n uint64) { if b.Containers.Size() == 0 { @@ -3973,7 +3984,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint header = pos if size > (1 << 16) { - err = fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + err = fmt.Errorf("it is logically impossible to have more than (1<<16) containers") return size, containerTyper, header, pos, haveRuns, err } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 993bcedb2..2dda414b3 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3224,11 +3224,11 @@ func TestContainerCombinations(t *testing.T) { //func getFunc(func(a, b *container) *container, m, n *container) *container { func runContainerFunc(f interface{}, c ...*Container) *Container { - switch f.(type) { + switch f := f.(type) { case func(*Container) *Container: - return f.(func(*Container) *Container)(c[0]) + return f(c[0]) case func(*Container, *Container) *Container: - return f.(func(a, b *Container) *Container)(c[0], c[1]) + return f(c[0], c[1]) } return nil } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 4c3d15a4d..b165042a9 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -37,6 +37,29 @@ func TestContainerCount(t *testing.T) { t.Fatalf("Count != CountRange\n") } } +func TestSize(t *testing.T) { + //array + a := roaring.NewFileBitmap(0, 65535, 131072) + if a.Size() != 6 { + t.Fatalf("Size in bytes incorrect \n") + } + + //bitmap + b := roaring.NewFileBitmap() + for i := uint64(0); i <= 4096; i++ { + b.DirectAdd(i) + } + + if b.Size() != 8192 { + t.Fatalf("Size in bytes incorrect \n") + } + //convert to rle + b.Optimize() + //rle + if b.Size() != 6 { + t.Fatalf("Size in bytes incorrect \n") + } +} func TestCountRange(t *testing.T) { tests := []struct { @@ -1498,6 +1521,41 @@ func BenchmarkSliceDescending(b *testing.B) { for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { bm.Add(col) } + bm.Add(0) + } +} + +func BenchmarkSliceAscendingStriped(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewFileBitmap() + l := uint64(pilosa.ShardWidth / 8) + for col := uint64(0); col < l; col++ { + bm.Add(l*0 + col) + bm.Add(l*1 + col) + bm.Add(l*2 + col) + bm.Add(l*3 + col) + bm.Add(l*4 + col) + bm.Add(l*5 + col) + bm.Add(l*6 + col) + bm.Add(l*7 + col) + } + } +} + +func BenchmarkSliceDescendingStriped(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewFileBitmap() + l := uint64(pilosa.ShardWidth / 8) + for col := uint64(l); col < l+1; col-- { + bm.Add(l*7 + col) + bm.Add(l*6 + col) + bm.Add(l*5 + col) + bm.Add(l*4 + col) + bm.Add(l*3 + col) + bm.Add(l*2 + col) + bm.Add(l*1 + col) + bm.Add(l*0 + col) + } } } diff --git a/server.go b/server.go index ca5bfcce4..4902c87d8 100644 --- a/server.go +++ b/server.go @@ -487,7 +487,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateShardMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field) + return fmt.Errorf("local field not found: %s/%s", obj.Index, obj.Field) } if err := f.AddRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { return errors.Wrap(err, "adding remote available shards") @@ -505,7 +505,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { - return fmt.Errorf("Local Index not found: %s", obj.Index) + return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta _, err := idx.createField(obj.Field, *opt) @@ -525,7 +525,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Field not found: %s", obj.Field) + return fmt.Errorf("local field not found: %s", obj.Field) } _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { @@ -534,7 +534,7 @@ func (s *Server) receiveMessage(m Message) error { case *DeleteViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Field not found: %s", obj.Field) + return fmt.Errorf("local field not found: %s", obj.Field) } err := f.deleteView(obj.View) if err != nil { diff --git a/server/cluster_test.go b/server/cluster_test.go index 90467f4b2..617bf6cad 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -399,7 +399,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { nodeID := mustNodeID(m0.URL()) resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator." + expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator" if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { diff --git a/server/server.go b/server/server.go index c80f8268e..5f99e510e 100644 --- a/server/server.go +++ b/server/server.go @@ -320,6 +320,7 @@ func (m *Command) setupNetworking() error { m.Config.Gossip, m.API, gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}), + gossip.WithPilosaLogger(m.logger), gossip.WithTransport(m.gossipTransport), ) if err != nil { diff --git a/server/server_test.go b/server/server_test.go index a2fb5b5d7..66e2cc112 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -293,7 +293,7 @@ func TestMain_GroupBy(t *testing.T) { } // Query row. - if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil { + if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil { t.Fatal(err) } else { test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) diff --git a/server_internal_test.go b/server_internal_test.go index e22681764..a8af8186c 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -38,7 +38,7 @@ func TestCountOpenFiles(t *testing.T) { func TestMonitorAntiEntropyZero(t *testing.T) { - td, err := ioutil.TempDir("", "") + td, err := ioutil.TempDir(*TempDir, "") if err != nil { t.Fatalf("getting temp dir: %v", err) } diff --git a/translate.go b/translate.go index b86041e3d..db129cd8f 100644 --- a/translate.go +++ b/translate.go @@ -372,7 +372,6 @@ func (s *TranslateFile) monitorReplication() { if err := s.replicate(ctx); err != nil { s.logger.Printf("pilosa: replication error: %s", err) } - select { case <-ctx.Done(): return @@ -412,22 +411,42 @@ func (s *TranslateFile) replicate(ctx context.Context) error { // Wrap in bufferred I/O so it implements io.ByteReader. bufr := bufio.NewReader(rc) + // we need a way to make an asynchronous routine hand us back an error, + // but we might not still be there to get it. so we have a buffer. + chErr := make(chan error, 1) + // Continually read new entries from primary and append to local store. for { // Read next available entry. var entry LogEntry - if _, err := entry.ReadFrom(bufr); err == io.EOF { + if _, err = entry.ReadFrom(bufr); err == io.EOF { return nil } else if err != nil { return err } - s.mu.Lock() - // Write to local store. - if err := s.appendEntry(&entry); err != nil { - s.mu.Unlock() - return err + // note: we should never end up spawning two of this goroutine + // at once. either we end up reading the error from chErr below, + // and this loop continues, or we don't, and the whole function + // returns. if the function returns, we can write that single + // error to the empty channel with a buffer of 1, the goroutine + // terminates, and chErr becomes garbage-collectable. + go func() { + s.mu.Lock() + defer s.mu.Unlock() + // Write to local store. + err = s.appendEntry(&entry) + chErr <- err + }() + select { + case err = <-chErr: + if err != nil { + return err + } + case <-s.replicationClosing: + return nil + case <-ctx.Done(): + return nil } - s.mu.Unlock() } } diff --git a/translate_test.go b/translate_test.go index fe78108ff..2b5d35a49 100644 --- a/translate_test.go +++ b/translate_test.go @@ -803,7 +803,7 @@ type TranslateFile struct { } func NewTranslateFile() *TranslateFile { - f, err := ioutil.TempFile("", "") + f, err := ioutil.TempFile(*TempDir, "") if err != nil { panic(err) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 21d8d0780..016d5276f 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -212,7 +212,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) t.common.Nodes = append(t.common.Nodes, node) // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i)) if err != nil { return nil, err } diff --git a/view_internal_test.go b/view_internal_test.go index 4ca7667ad..6867b03ed 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -24,7 +24,7 @@ import ( // mustOpenView returns a new instance of View with a temporary path. func mustOpenView(index, field, name string) *view { - path, err := ioutil.TempDir("", "pilosa-view-") + path, err := ioutil.TempDir(*TempDir, "pilosa-view-") if err != nil { panic(err) }