diff --git a/Makefile b/Makefile index e7af8e119..55648c7f1 100644 --- a/Makefile +++ b/Makefile @@ -139,9 +139,9 @@ gometalinter: require-gometalinter --enable=ineffassign \ --enable=interfacer \ --enable=maligned \ - --enable=megacheck \ --enable=misspell \ --enable=nakedret \ + --enable=staticcheck \ --enable=unconvert \ --enable=unparam \ --enable=vet \ diff --git a/cluster.go b/cluster.go index e966c57a1..4565e91f3 100644 --- a/cluster.go +++ b/cluster.go @@ -609,8 +609,8 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool { // Nodes returns a copy of the slice of nodes in the cluster. Safe for // concurrent use, result may be modified. func (c *cluster) Nodes() []*Node { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() ret := make([]*Node, len(c.nodes)) copy(ret, c.nodes) return ret @@ -1792,7 +1792,7 @@ func (c *cluster) nodeLeave(nodeID string) error { } if c.state != ClusterStateNormal && c.state != ClusterStateDegraded { - return fmt.Errorf("Cluster must be in state %s to remove a node. Current state: %s", + return fmt.Errorf("cluster must be '%s' to remove a node but is '%s'", ClusterStateNormal, c.state) } @@ -1803,7 +1803,7 @@ func (c *cluster) nodeLeave(nodeID string) error { // Prevent removing the coordinator node (this node). if nodeID == c.Node.ID { - return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.") + return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator") } // See if resize job can be generated diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 50602d385..a6cf437b2 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -83,7 +83,7 @@ func TestFragCombos(t *testing.T) { // newIndexWithTempPath returns a new instance of Index. func newIndexWithTempPath(name string) *Index { - path, err := ioutil.TempDir("", "pilosa-index-") + path, err := ioutil.TempDir(*TempDir, "pilosa-index-") if err != nil { panic(err) } diff --git a/ctl/import.go b/ctl/import.go index ab19064ce..c120c955f 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -151,11 +151,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) if err != nil { - return fmt.Errorf("Error Creating Index: %s", err) + return errors.Wrap(err, "creating index") } err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.FieldOptions) if err != nil { - return fmt.Errorf("Error Creating Field: %s", err) + return errors.Wrap(err, "creating field") } return nil } diff --git a/ctl/server.go b/ctl/server.go index 5ed40dbf4..441f2b1e7 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") + flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging") @@ -49,6 +50,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") + flags.StringVarP(&srv.Config.Gossip.AdvertiseHost, "gossip.advertise-host", "", srv.Config.Gossip.AdvertiseHost, "Host on which memberlist should advertise.") + flags.StringVarP(&srv.Config.Gossip.AdvertisePort, "gossip.advertise-port", "", srv.Config.Gossip.AdvertisePort, "Port on which memberlist should advertise.") + flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.") diff --git a/diagnostics.go b/diagnostics.go index 5ed97940a..1eb3f2606 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -137,11 +137,11 @@ func (d *diagnosticsCollector) compareVersion(value string) error { localVersion := versionSegments(d.version) if localVersion[0] < currentVersion[0] { //Major - return fmt.Errorf("Warning: You are running Pilosa %s. A newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) + return fmt.Errorf("you are running Pilosa %s, a newer version (%s) is available: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[1] < currentVersion[1] && localVersion[0] == currentVersion[0] { // Minor - return fmt.Errorf("Warning: You are running Pilosa %s. The latest Minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) + return fmt.Errorf("you are running Pilosa %s, the latest minor release is %s: https://github.com/pilosa/pilosa/releases", d.version, value) } else if localVersion[2] < currentVersion[2] && localVersion[0] == currentVersion[0] && localVersion[1] == currentVersion[1] { // Patch - return fmt.Errorf("There is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) + return fmt.Errorf("there is a new patch release of Pilosa available: %s: https://github.com/pilosa/pilosa/releases", value) } return nil diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index f1536e1d1..99c0a83ac 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -82,19 +82,19 @@ func TestDiagnosticsVersion_Compare(t *testing.T) { d.SetVersion(version) err := d.compareVersion("v1.7.0") - if !strings.Contains(err.Error(), "A newer version") { + if !strings.Contains(err.Error(), "a newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.compareVersion("1.7.0") - if !strings.Contains(err.Error(), "A newer version") { + if !strings.Contains(err.Error(), "a newer version") { t.Fatalf("Expected a newer version is available, actual error: %s", err) } err = d.compareVersion("0.7.0") - if !strings.Contains(err.Error(), "The latest Minor release is") { + if !strings.Contains(err.Error(), "the latest minor release is") { t.Fatalf("Expected Minor Version Missmatch, actual error: %s", err) } err = d.compareVersion("0.1.2") - if !strings.Contains(err.Error(), "There is a new patch release of Pilosa") { + if !strings.Contains(err.Error(), "there is a new patch release of Pilosa") { t.Fatalf("Expected Patch Version Missmatch, actual error: %s", err) } err = d.compareVersion("0.1.1") diff --git a/docs/configuration.md b/docs/configuration.md index 140258437..1432cbea1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -36,6 +36,17 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h ### All Options +#### Advertise + +* Description: Address advertised by the server to other nodes in the cluster and to clients via the `/status` endpoint. Host defaults to the IP address represented by `bind` and port to 10101. If `bind` is set to `0.0.0.0` and `advertise` is not specified, then Pilosa will try to determine a reasonable, external IP address to use for `advertise`. +* Flag: `--advertise="192.168.1.100:10101"` +* Env: `PILOSA_BIND="192.168.1.100:10101"` +* Config: + + ```toml + advertise = 192.168.1.100:10101 + ``` + #### Anti Entropy Interval * Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync. @@ -50,7 +61,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Bind -* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. +* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. If `bind` is set to `0.0.0.0` then Pilosa will listen on all available interfaces. * Flag: `--bind="localhost:10101"` * Env: `PILOSA_BIND="localhost:10101"` * Config: @@ -115,6 +126,30 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h max-writes-per-request = 5000 ``` +#### Gossip Advertise Host + +* Description: Host on which memberlist should advertise. Defaults to `advertise` host. +* Flag: `--gossip.advertise-host=192.168.1.100` +* Env: `PILOSA_GOSSIP_ADVERTISE_HOST=192.168.1.100 +* Config: + + ```toml + [gossip] + advertise-host = 192.168.1.100 + ``` + +#### Gossip Advertise Port + +* Description: Port on which memberlist should advertise. Defaults to `advertise` port. +* Flag: `--gossip.advertise-port=15001` +* Env: `PILOSA_GOSSIP_ADVERTISE_PORT=15001` +* Config: + + ```toml + [gossip] + advertise-port = 15001 + ``` + #### Gossip Port * Description: Port to which Pilosa should bind for internal communication. If more than one Pilosa server is running on the same host, the gossip port for each server must be unique. @@ -366,7 +401,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -379,7 +414,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -392,7 +427,7 @@ A three node cluster running on different hosts could be minimally configured as [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] [cluster] replicas = 1 @@ -410,7 +445,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -428,7 +463,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -446,7 +481,7 @@ The same cluster which uses HTTPS instead of HTTP can be configured as follows. [gossip] port = 12000 - seed = "node0.pilosa.com:12000" + seeds = ["node0.pilosa.com:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -468,7 +503,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12000 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -486,7 +521,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12001 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] @@ -504,7 +539,7 @@ You can run a cluster on the same host using the configuration above with a few [gossip] port = 12002 - seed = "localhost:12000" + seeds = ["localhost:12000"] key = "/home/pilosa/private/gossip.key32" [cluster] diff --git a/docs/query-language.md b/docs/query-language.md index 73e1cab7d..4dd838727 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -594,6 +594,34 @@ Count(Row(stargazer=1)) * Result is the number of repositories that user 1 has starred. +#### Shift +**Spec:** + +``` +Shift(, [n=UINT]) +``` + +**Description:** + +Returns the row specified by `ROW_CALL` shifted by `n` bits. + +**Result Type:** object with attrs and columns + +attrs will always be empty + +**Examples:** + +Query all columns with a bit set in row 1 of the field `stargazer` +and shift the result by 2: +```request +Shift(Row(stargazer=1), n=2) +``` +```response +{"attrs":{},"columns":[12, 22]} +``` + +* columns are the repositories which user 1 has starred shifted by 2 bits. + #### TopN **Spec:** @@ -786,7 +814,7 @@ Options(Row(f1=10), shards=[0, 2]) **Spec:** ``` -Rows(field=, previous=, limit=, column=) +Rows(, previous=, limit=, column=) ``` **Description:** @@ -810,7 +838,7 @@ result of the previous request will start from the next available row. Without keys: ```request -Rows(field=blah) +Rows(blah) ``` ```response {"rows":[1,9,39]} @@ -818,7 +846,7 @@ Rows(field=blah) With keys: ```request -Rows(field=blahk) +Rows(blahk) ``` ```response {"rows":null,"keys":["haha","zaaa","traa"]} @@ -859,7 +887,7 @@ specify the field and row for each row that was intersected to get that result. A single `Rows` query. ```request -GroupBy(Rows(field=blah)) +GroupBy(Rows(blah)) ``` ```response [{"group":[{"field":"blah","rowID":1}],"count":1}, @@ -869,7 +897,7 @@ GroupBy(Rows(field=blah)) With two `Rows` queries - one with IDs and one with keys. ```request -GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) +GroupBy(Rows(blah), Rows(blahk), limit=7) ``` ```response [{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1}, @@ -883,7 +911,7 @@ GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) Getting the rest of the results from the previous example (paging). ```request -GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7) +GroupBy(Rows(blah, previous=39), Rows(blahk, previous="haha"), limit=7) ``` ```response diff --git a/executor.go b/executor.go index f68fec726..a5c9b4391 100644 --- a/executor.go +++ b/executor.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "log" "sort" "time" @@ -559,6 +558,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeXorShard(ctx, index, c, shard) case "Not": return e.executeNotShard(ctx, index, c, shard) + case "Shift": + return e.executeShiftShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -806,7 +807,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca return nil, nil } - if minThreshold <= 0 { + if minThreshold == 0 { minThreshold = defaultMinThreshold } @@ -915,6 +916,12 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // TODO support TopN in here would be really cool - and pretty easy I think. childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { + // Check "field" first for backwards compatibility, then set _field. + // TODO: remove at Pilosa 2.0 + if fieldName, ok := child.Args["field"].(string); ok { + child.Args["_field"] = fieldName + } + if child.Name != "Rows" { return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name) } @@ -1089,6 +1096,17 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + // Fetch field name from argument. + // Check "field" first for backwards compatibility. + // TODO: remove at Pilosa 2.0 + var fieldName string + var ok bool + if fieldName, ok = c.Args["field"].(string); ok { + c.Args["_field"] = fieldName + } + if fieldName, ok = c.Args["_field"].(string); !ok { + return nil, errors.New("Rows() field required") + } if columnID, ok, err := c.UintArg("column"); err != nil { return nil, errors.Wrap(err, "getting column") } else if ok { @@ -1097,7 +1115,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeRowsShard(ctx, index, c, shard) + return e.executeRowsShard(ctx, index, fieldName, c, shard) } // Determine limit so we can use it when reducing. @@ -1122,22 +1140,25 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - // Fetch field name from argument. - fieldName, ok := c.Args["field"].(string) - if !ok { - return nil, errors.New("Rows() argument required: field") - } // Fetch field. f := e.Holder.Field(index, fieldName) if f == nil { return nil, ErrFieldNotFound } + + // Rows query does not currently support a `time` field that has + // `noStandardView: true`. + // TODO https://github.com/pilosa/pilosa/issues/1783 + if f.Type() == FieldTypeTime && f.options.NoStandardView { + return nil, errors.New("Rows() query on time field with no standard view is not currently supported") + } + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { return make(RowIDs, 0), nil @@ -1174,7 +1195,7 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal defer span.Finish() if c.Name == "Range" { - log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.") + e.Holder.Logger.Printf("DEPRECATED: Range() is deprecated, please use Row() instead.") } // Handle bsiGroup ranges differently. @@ -1508,6 +1529,27 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeShiftShard executes a shift() call for a local shard. +func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + n, _, err := c.IntArg("n") + if err != nil { + return nil, fmt.Errorf("executeShiftShard: %v", err) + } + + if len(c.Children) == 0 { + return nil, errors.New("Shift() requires an input row") + } else if len(c.Children) > 1 { + return nil, errors.New("Shift() only accepts a single row input") + } + + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + + return row.Shift(n) +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") @@ -1568,14 +1610,14 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if err != nil { return false, fmt.Errorf("reading Clear() row: %v", err) } else if !ok { - return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel) + return false, fmt.Errorf("row= argument required to Clear() call") } colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { return false, fmt.Errorf("reading Clear() column: %v", err) } else if !ok { - return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel) + return false, fmt.Errorf("column argument to Clear(, =) required") } return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) @@ -1694,19 +1736,19 @@ func (e *executor) executeClearRowShard(ctx context.Context, index string, c *pq return changed, nil } -// executeSetRow executes a SetRow() call. +// executeSetRow executes a Store() call. func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { - // Ensure the field type supports SetRow(). + // Ensure the field type supports Store(). fieldName, err := c.FieldArg() if err != nil { - return false, errors.New("SetRow() argument required: field") + return false, errors.New("field required for Store()") } field := e.Holder.Field(index, fieldName) if field == nil { return false, ErrFieldNotFound } if field.Type() != FieldTypeSet { - return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) + return false, fmt.Errorf("can't Store() on a %s field", field.Type()) } // Execute calls in bulk on each remote node and merge. @@ -1731,15 +1773,15 @@ func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { fieldName, err := c.FieldArg() if err != nil { - return false, errors.New("SetRow() argument required: field") + return false, errors.New("Store() argument required: field") } // Read fields using labels. rowID, ok, err := c.UintArg(fieldName) if err != nil { - return false, fmt.Errorf("reading SetRow() row: %v", err) + return false, fmt.Errorf("reading Store() row: %v", err) } else if !ok { - return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel) + return false, fmt.Errorf("need the = argument on Store()") } field := e.Holder.Field(index, fieldName) @@ -1756,7 +1798,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. } src = row } else { - return false, errors.New("SetRow() requires a source row") + return false, errors.New("Store() requires a source row") } // Set the row on the standard view. @@ -1775,7 +1817,7 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. } set, err := fragment.setRow(src, rowID) if err != nil { - return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) + return false, errors.Wrapf(err, "storing row %d on view %s shard %d", rowID, viewStandard, shard) } changed = changed || set @@ -2336,7 +2378,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") case "Rows": - fieldName = callArgString(c, "field") + fieldName = callArgString(c, "_field") rowKey = "previous" colKey = "column" case "GroupBy": @@ -2441,7 +2483,7 @@ func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) e fields := make([]*Field, len(c.Children)) for i, child := range c.Children { - fieldname := callArgString(child, "field") + fieldname := callArgString(child, "_field") field := idx.Field(fieldname) if field == nil { return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) @@ -2552,7 +2594,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res case RowIDs: other := RowIdentifiers{} - fieldName := callArgString(call, "field") + fieldName := callArgString(call, "_field") if fieldName == "" { return nil, ErrFieldNotFound } @@ -2750,11 +2792,12 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde fields: make([]FieldRow, len(children)), } + var fieldName string + var ok bool ignorePrev := false for i, call := range children { - fieldName, ok := call.Args["field"].(string) - if !ok { - return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"]) + if fieldName, ok = call.Args["_field"].(string); !ok { + return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["_field"]) } if holder.Field(index, fieldName) == nil { return nil, ErrFieldNotFound diff --git a/executor_internal_test.go b/executor_internal_test.go index 629d704db..4177968d5 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -14,7 +14,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { e := &executor{ Holder: NewHolder(), } - e.Holder.Path, _ = ioutil.TempDir("", "") + e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") err := e.Holder.Open() if err != nil { t.Fatalf("opening holder: %v", err) @@ -46,7 +46,7 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { t.Fatalf("translating rows %v, %v", erra, errb) } - query, err := pql.ParseString(`GroupBy(Rows(field=ak), Rows(field=b), Rows(field=ck), previous=["la", 0, "ha"])`) + query, err := pql.ParseString(`GroupBy(Rows(ak), Rows(b), Rows(ck), previous=["la", 0, "ha"])`) if err != nil { t.Fatalf("parsing query: %v", err) } @@ -69,28 +69,28 @@ func TestExecutor_TranslateGroupByCall(t *testing.T) { err string }{ { - pql: `GroupBy(Rows(field=notfound), previous=1)`, + pql: `GroupBy(Rows(notfound), previous=1)`, err: "'previous' argument must be list", }, { - pql: `GroupBy(Rows(field=ak), previous=["la", 0])`, + pql: `GroupBy(Rows(ak), previous=["la", 0])`, err: "mismatched lengths", }, { - pql: `GroupBy(Rows(field=ak), previous=[1])`, + pql: `GroupBy(Rows(ak), previous=[1])`, err: "prev value must be a string", }, { - pql: `GroupBy(Rows(field=notfound), previous=[1])`, + pql: `GroupBy(Rows(notfound), previous=[1])`, err: ErrFieldNotFound.Error(), }, // TODO: an unknown key will actually allocate an id. this is probably bad. // { - // pql: `GroupBy(Rows(field=ak), previous=["zoop"])`, + // pql: `GroupBy(Rows(ak), previous=["zoop"])`, // err: "translating row key '", // }, { - pql: `GroupBy(Rows(field=b), previous=["la"])`, + pql: `GroupBy(Rows(b), previous=["la"])`, err: "which doesn't use string keys", }, } diff --git a/executor_test.go b/executor_test.go index 0d42af8c5..3d17a14e9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -16,7 +16,9 @@ package pilosa_test import ( "context" + "flag" "fmt" + "io/ioutil" "math/rand" "reflect" "strconv" @@ -33,6 +35,22 @@ import ( "github.com/pkg/errors" ) +var ( + TempDir = getTempDirString() +) + +func getTempDirString() (td *string) { + tdflag := flag.Lookup("temp-dir") + + if tdflag == nil { + td = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") + } else { + s := tdflag.Value.String() + td = &s + } + return td +} + // Ensure a row query can be executed. func TestExecutor_Execute_Row(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { @@ -2283,7 +2301,7 @@ Set(4500001, fn=4) t.Run("remote groupBy", func(t *testing.T) { if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", - Query: `GroupBy(Rows(field=f))`, + Query: `GroupBy(Rows(f))`, }); err != nil { t.Fatalf("GroupBy querying: %v", err) } else { @@ -2987,7 +3005,16 @@ func TestExecutor_Execute_SetRow(t *testing.T) { } func benchmarkExistence(nn bool, b *testing.B) { - c := test.MustRunCluster(b, 1) + c := test.MustNewCluster(b, 1) + var err error + c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkExistence") + if err != nil { + b.Fatalf("getting temp dir: %v", err) + } + err = c.Start() + if err != nil { + b.Fatalf("starting cluster: %v", err) + } defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -3038,27 +3065,45 @@ func TestExecutor_Execute_Rows(t *testing.T) { {13, 3}, }) - rows := c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) + rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, limit=2)`).Results[0].(pilosa.RowIdentifiers) + // backwards compatibility + // TODO: remove at Pilosa 2.0 + rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { + t.Fatalf("unexpected rows: %+v", rows) + } + + rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) + rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } - rows = c.Query(t, "i", `Rows(field=general, column=2)`).Results[0].(pilosa.RowIdentifiers) + rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } } +func TestExecutor_Execute_RowsTime(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "t", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + + exp := "executing: Rows() query on time field with no standard view is not currently supported" + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=t)`}); err == nil || err.Error() != exp { + t.Fatalf("expected error: %s", exp) + } +} + func TestExecutor_Execute_Query_Error(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -3070,35 +3115,27 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { }{ { query: "GroupBy(Rows())", - error: "Rows call must have 'field' argument", + error: "Rows call must have field", }, { - query: "GroupBy(Rows(field=true))", - error: "Rows call must have 'field' argument", + query: "GroupBy(Rows(\"true\"))", + error: "parsing: parsing:", }, { - query: "GroupBy(Rows(field=\"true\"))", - error: "field not found", + query: "GroupBy(Rows(1))", + error: "parsing: parsing:", }, { - query: "GroupBy(Rows(field=1))", - error: "Rows call must have 'field' argument", - }, - { - query: "GroupBy(Rows(field))", - error: "parse error", - }, - { - query: "GroupBy(Rows(field=general, limit=-1))", + query: "GroupBy(Rows(general, limit=-1))", error: "must be positive, but got", }, { - query: "GroupBy(Rows(field=general), limit=-1)", + query: "GroupBy(Rows(general), limit=-1)", error: "must be positive, but got", }, { - query: "GroupBy(Rows(field=general), filter=Rows(field=general))", - error: "unknown call: Rows", + query: "GroupBy(Rows(general), filter=Rows(general))", + error: "parsing: parsing:", }, } @@ -3112,7 +3149,7 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r) } if !strings.Contains(err.Error(), test.error) { - t.Fatalf("unexpected error message: %s", err.Error()) + t.Fatalf("unexpected error message:\n%s != %s", test.error, err.Error()) } }) } @@ -3158,68 +3195,80 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { q string exp []string }{ + { + q: `Rows(f)`, + exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"}, + }, + // backwards compatibility + // TODO: remove at Pilosa 2.0 { q: `Rows(field=f)`, exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"}, }, + { + q: `Rows(f, limit=2)`, + exp: []string{"0", "1"}, + }, + // backwards compatibility + // TODO: remove at Pilosa 2.0 { q: `Rows(field=f, limit=2)`, exp: []string{"0", "1"}, }, { - q: `Rows(field=f, previous="15")`, + q: `Rows(f, previous="15")`, exp: []string{"16", "17", "18"}, }, { - q: `Rows(field=f, previous="11", limit=2)`, + q: `Rows(f, previous="11", limit=2)`, exp: []string{"12", "13"}, }, { - q: `Rows(field=f, previous="17", limit=5)`, + q: `Rows(f, previous="17", limit=5)`, exp: []string{"18"}, }, { - q: `Rows(field=f, previous="18")`, + q: `Rows(f, previous="18")`, exp: []string{}, }, { - q: `Rows(field=f, previous="1", limit=0)`, + q: `Rows(f, previous="1", limit=0)`, exp: []string{}, }, { - q: `Rows(field=f, column="1")`, + q: `Rows(f, column="1")`, exp: []string{"0", "1"}, }, { - q: `Rows(field=f, column="2")`, + q: `Rows(f, column="2")`, exp: []string{"0", "1", "2"}, }, { - q: `Rows(field=f, column="3")`, + q: `Rows(f, column="3")`, exp: []string{"1", "2", "3"}, }, { - q: `Rows(field=f, limit=2, column="3")`, + q: `Rows(f, limit=2, column="3")`, exp: []string{"1", "2"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="15", column="%d")`, ShardWidth*9+17), + q: fmt.Sprintf(`Rows(f, previous="15", column="%d")`, ShardWidth*9+17), exp: []string{"16", "17"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column="%d")`, ShardWidth*5+14), + q: fmt.Sprintf(`Rows(f, previous="11", limit=2, column="%d")`, ShardWidth*5+14), exp: []string{"12", "13"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column="%d")`, ShardWidth*9+18), + q: fmt.Sprintf(`Rows(f, previous="17", limit=5, column="%d")`, ShardWidth*9+18), exp: []string{"18"}, }, { - q: `Rows(field=f, previous="18", column="19")`, + q: `Rows(f, previous="18", column="19")`, exp: []string{}, }, { - q: `Rows(field=f, previous="1", limit=0, column="0")`, + q: `Rows(f, previous="1", limit=0, column="0")`, exp: []string{}, }, } @@ -3271,13 +3320,27 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("Unknown Field ", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(missing))`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) } } }) + // backwards compatibility + // TODO: remove at Pilosa 2.0 + t.Run("BasicLegacy", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(sub))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + t.Run("Basic", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, @@ -3286,7 +3349,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3296,7 +3359,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3306,7 +3369,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3315,7 +3378,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3336,7 +3399,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3364,7 +3427,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test wrapping with previous", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, @@ -3374,14 +3437,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test previous is last result", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa, previous=3), Rows(field=wb, previous=3), Rows(field=wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) if len(results) > 0 { t.Fatalf("expected no results because previous specified last result") } }) t.Run("test wrapping multiple", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } @@ -3405,7 +3468,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {3, ShardWidth}, }) t.Run("distinct rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, @@ -3417,7 +3480,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, @@ -3428,7 +3491,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { - results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(field=ma), Rows(field=mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, @@ -3453,7 +3516,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {1, ShardWidth}, }) t.Run("same rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=na), Rows(field=nb))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2}, @@ -3490,11 +3553,11 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("test wrapping with previous", func(t *testing.T) { totalResults := make([]pilosa.GroupCount, 0) - results := c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount) totalResults = append(totalResults, results...) for len(totalResults) < 64 { lastGroup := results[len(results)-1].Group - query := fmt.Sprintf("GroupBy(Rows(field=ppa, previous=%d), Rows(field=ppb, previous=%d), Rows(field=ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) + query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount) totalResults = append(totalResults, results...) } @@ -3537,7 +3600,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount) test.CheckGroupBy(t, expected, results) }) @@ -3551,7 +3614,16 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } func BenchmarkGroupBy(b *testing.B) { - c := test.MustRunCluster(b, 1) + c := test.MustNewCluster(b, 1) + var err error + c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkGroupBy") + if err != nil { + b.Fatalf("getting temp dir: %v", err) + } + err = c.Start() + if err != nil { + b.Fatalf("starting cluster: %v", err) + } defer c.Close() c.CreateField(b, "i", pilosa.IndexOptions{}, "a") c.CreateField(b, "i", pilosa.IndexOptions{}, "b") @@ -3586,7 +3658,7 @@ func BenchmarkGroupBy(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c))`) + c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c))`) } }) @@ -3594,7 +3666,7 @@ func BenchmarkGroupBy(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c), limit=4)`) + c.Query(b, "i", `GroupBy(Rows(a), Rows(b), Rows(c), limit=4)`) } }) @@ -3644,3 +3716,91 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt return responses } + +func TestExecutor_Execute_Shift(t *testing.T) { + t.Run("Shift Bit 0", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift container boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 65535) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift shard boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + orig := []uint64{1, ShardWidth - 1, ShardWidth + 1} + shift1 := []uint64{2, ShardWidth, ShardWidth + 2} + shift2 := []uint64{3, ShardWidth + 1, ShardWidth + 3} + + for _, bit := range orig { + hldr.SetBit("i", "general", 10, bit) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift1) { + t.Fatalf("unexpected shift by 1: expected: %+v, but got: %+v", shift1, columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=2)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift2) { + t.Fatalf("unexpected shift by 2: expected: %+v, but got: %+v", shift2, columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, orig) { + t.Fatalf("unexpected shift by 0: expected: %+v, but got: %+v", orig, columns) + } + }) + + t.Run("Shift shard boundary no create", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1 + hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth + hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 + hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 + + exp := []uint64{ShardWidth - 1, ShardWidth, ShardWidth + 1, ShardWidth + 3} + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2, ShardWidth + 4}) { + t.Fatalf("unexpected columns: \n%+v\n%+v", columns, exp) + } + }) +} diff --git a/field.go b/field.go index 669700574..38ba8ee76 100644 --- a/field.go +++ b/field.go @@ -476,13 +476,21 @@ func (f *Field) saveMeta() error { // applyOptions configures the field based on opt. func (f *Field) applyOptions(opt FieldOptions) error { switch opt.Type { - case FieldTypeSet, "": - f.options.Type = FieldTypeSet + case FieldTypeSet, FieldTypeMutex, "": + fldType := opt.Type + if fldType == "" { + fldType = FieldTypeSet + } + f.options.Type = fldType if opt.CacheType != "" { f.options.CacheType = opt.CacheType } if opt.CacheSize != 0 { - f.options.CacheSize = opt.CacheSize + if opt.CacheType == CacheTypeNone { + f.options.CacheSize = 0 + } else { + f.options.CacheSize = opt.CacheSize + } } f.options.Min = 0 f.options.Max = 0 @@ -524,18 +532,6 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.Close() return errors.Wrap(err, "setting time quantum") } - case FieldTypeMutex: - f.options.Type = FieldTypeMutex - if opt.CacheType != "" { - f.options.CacheType = opt.CacheType - } - if opt.CacheSize != 0 { - f.options.CacheSize = opt.CacheSize - } - f.options.Min = 0 - f.options.Max = 0 - f.options.TimeQuantum = "" - f.options.Keys = opt.Keys case FieldTypeBool: f.options.Type = FieldTypeBool f.options.CacheType = CacheTypeNone diff --git a/field_internal_test.go b/field_internal_test.go index c2495e9c0..0a26225bb 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -192,7 +192,7 @@ type TestField struct { // NewTestField returns a new instance of TestField d/0. func NewTestField(opts FieldOption) *TestField { - path, err := ioutil.TempDir("", "pilosa-field-") + path, err := ioutil.TempDir(*TempDir, "pilosa-field-") if err != nil { panic(err) } diff --git a/fragment.go b/fragment.go index 4f71809c9..58680e655 100644 --- a/fragment.go +++ b/fragment.go @@ -1050,7 +1050,7 @@ func (f *fragment) top(opt topOptions) ([]Pair, error) { rowID, cnt := pair.ID, pair.Count // Ignore empty rows. - if cnt <= 0 { + if cnt == 0 { continue } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index afd04616f..548e5f40a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -39,12 +39,9 @@ var ( // In order to generate the sample fragment file, // run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") - TempDir = "" -) -func init() { // nolint: gochecknoinits - flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") -} + TempDir = flag.String("temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") +) // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { @@ -2069,7 +2066,7 @@ func BenchmarkFileWrite(b *testing.B) { b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, err := ioutil.TempFile(TempDir, "") + f, err := ioutil.TempFile(*TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } @@ -2125,7 +2122,7 @@ func (f *fragment) CleanKeep(t testing.TB) { // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { - file, err := ioutil.TempFile(TempDir, "pilosa-fragment-") + file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") if err != nil { panic(err) } diff --git a/gossip/gossip.go b/gossip/gossip.go index 491e14cf2..1eee9dc3d 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "log" "net" + "os" "strconv" "strings" "sync" @@ -50,8 +51,11 @@ type memberSet struct { Logger logger.Logger - logger *log.Logger + // stdLogger is only used when passed into memberlist library things that take a std library logger rather than an interface. + stdLogger *log.Logger + // logOutput is similar to stdLogger in that it's passed to memberlist things which can't take a pilosa Logger. logOutput io.Writer + transport *Transport eventReceiver *eventReceiver @@ -151,14 +155,20 @@ func WithTransport(transport *Transport) memberSetOption { } } -// WithLogger is a functional option for providing a logger to NewMemberSet. +// WithLogger is a functional option for providing a Go logger to NewMemberSet. +// If the memberSet's transport is nil, this logger will be used when creating +// one. If WithLogOutput is not used, this logger will be passed to memberlist +// for it to use internally. This logger is not used for logging by code in this +// (gossip) package - for that, use the WithPilosaLogger option. func WithLogger(logger *log.Logger) memberSetOption { return func(g *memberSet) error { - g.logger = logger + g.stdLogger = logger return nil } } +// WithLogOutput allows one to pass a Writer which will in turn be passed to +// memberlist for use in logging. func WithLogOutput(o io.Writer) memberSetOption { return func(g *memberSet) error { g.logOutput = o @@ -166,7 +176,20 @@ func WithLogOutput(o io.Writer) memberSetOption { } } -// NewMemberSet returns a new instance of GossipMemberSet based on options. +// WithPilosaLogger allows one to configure a memberSet with a logger of their +// choice which satisfies the pilosa logger interface. +func WithPilosaLogger(l logger.Logger) memberSetOption { + return func(g *memberSet) error { + g.Logger = l + return nil + } +} + +// NewMemberSet returns a new instance of GossipMemberSet based on options. The +// logging options which can be passed to NewMemberSet are complicated for +// historical reasons - please pass WithPilosaLogger, and either WithLogOutput +// or WithLogger. If you pass WithLogOutput, be sure to also pass in a Transport +// using WithTransport. func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) { host := api.Node().URI.Host g := &memberSet{ @@ -180,7 +203,8 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem return nil, errors.Wrap(err, "executing option") } } - ger := newEventReceiver(g.logger, api) + + ger := newEventReceiver(g.Logger, api) g.eventReceiver = ger if g.transport == nil { @@ -189,8 +213,16 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem return nil, fmt.Errorf("convert port: %s", err) } + if g.stdLogger == nil { + if g.logOutput != nil { + g.stdLogger = logger.NewStandardLogger(g.logOutput).Logger() + } else { + g.stdLogger = log.New(os.Stderr, "", log.LstdFlags) + } + } + // Set up the transport. - transport, err := NewTransport(host, port, g.logger) + transport, err := NewTransport(host, port, g.stdLogger) if err != nil { return nil, fmt.Errorf("new tranport: %s", err) } @@ -209,14 +241,29 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem } } + //////////////////// // memberlist config conf := memberlist.DefaultWANConfig() conf.Transport = g.transport.net conf.Name = api.Node().ID conf.BindAddr = api.Node().URI.Host conf.BindPort = port - conf.AdvertisePort = port - conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) + // AdvertisePort + if cfg.AdvertisePort != "" { + if p, err := strconv.Atoi(cfg.Port); err != nil { + return nil, fmt.Errorf("convert advertise port: %s", err) + } else { + conf.AdvertisePort = p + } + } else { + conf.AdvertisePort = port + } + // AdvertiseHost + if cfg.AdvertiseHost != "" { + conf.AdvertiseAddr = cfg.AdvertiseHost + } else { + conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) + } // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -233,7 +280,7 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem if g.logOutput != nil { conf.LogOutput = g.logOutput } else { - conf.Logger = g.logger + conf.Logger = g.stdLogger } g.config = &config{ @@ -318,11 +365,11 @@ type eventReceiver struct { ch chan memberlist.NodeEvent papi *pilosa.API - logger *log.Logger + logger logger.Logger } // newEventReceiver returns a new instance of GossipEventReceiver. -func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { +func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver { ger := &eventReceiver{ ch: make(chan memberlist.NodeEvent, 1), logger: logger, @@ -468,7 +515,7 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { nt, err := makeNetRetry(limit) if err != nil { - return nil, fmt.Errorf("Could not set up network transport: %v", err) + return nil, errors.Wrap(err, "could not set up network transport") } return nt, nil @@ -477,7 +524,16 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { // Config holds toml-friendly memberlist configuration. type Config struct { // Port indicates the port to which pilosa should bind for internal state sharing. - Port string `toml:"port"` + Port string `toml:"port"` + + // AdvertiseHost is the hostname or IP other nodes should use to connect to + // this host. If left blank, the value for Host will be used. This is useful + // in some proxy and NAT scenarios. + AdvertiseHost string `toml:"advertise-host"` + // AdvertisePort is the port other nodes will use to connect to this one. + // Behaves like AdvertiseHost. + AdvertisePort string `toml:"advertise-port"` + Seeds []string `toml:"seeds"` Key string `toml:"key"` // StreamTimeout is the timeout for establishing a stream connection with diff --git a/holder_internal_test.go b/holder_internal_test.go index 1c4dfad88..b13df7809 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -46,7 +46,7 @@ func (h *tHolder) Reopen() error { } func newHolder() *tHolder { - path, err := ioutil.TempDir("", "pilosa-") + path, err := ioutil.TempDir(*TempDir, "pilosa-") if err != nil { panic(err) } diff --git a/http/client.go b/http/client.go index 78382b81b..92f6269fe 100644 --- a/http/client.go +++ b/http/client.go @@ -1015,7 +1015,7 @@ func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, erro if resp != nil { resp.Body.Close() } - return nil, errors.Wrap(err, "executing request") + return nil, errors.Wrap(err, "getting response") } if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() diff --git a/http/handler.go b/http/handler.go index b85be6b0e..8cc059035 100644 --- a/http/handler.go +++ b/http/handler.go @@ -584,11 +584,11 @@ func validateOptions(data map[string]interface{}, validIndexOptions []string) er } for kk, vv := range options { if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) + return fmt.Errorf("unknown key: %v:%v", kk, vv) } } default: - return fmt.Errorf("Unknown key: %v:%v", k, v) + return fmt.Errorf("unknown key: %v:%v", k, v) } } return nil diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index ecbf8ff87..0cb90655f 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -35,8 +35,8 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, - {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, + {json: `{"option": {}}`, err: "unknown key: option:map[]"}, + {json: `{"options": {"badKey": "test"}}`, err: "unknown key: badKey:test"}, } for _, test := range tests { actual := &postIndexRequest{} diff --git a/index_internal_test.go b/index_internal_test.go index fd83fd775..dc0296049 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -21,7 +21,7 @@ import ( // mustOpenIndex returns a new, opened index at a temporary path. Panic on error. func mustOpenIndex(opt IndexOptions) *Index { - path, err := ioutil.TempDir("", "pilosa-index-") + path, err := ioutil.TempDir(*TempDir, "pilosa-index-") if err != nil { panic(err) } diff --git a/lru/lru.go b/lru/lru.go index d7bb1929c..ba0121a9a 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -83,7 +83,7 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { } // remove removes the provided key from the cache. -func (c *Cache) remove(key Key) { // nolint: megacheck +func (c *Cache) remove(key Key) { // nolint: staticcheck if c.cache == nil { return } @@ -121,7 +121,7 @@ func (c *Cache) Len() int { } // clear purges all stored items from the cache. -func (c *Cache) clear() { // nolint: megacheck +func (c *Cache) clear() { // nolint: staticcheck if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) diff --git a/pilosa.go b/pilosa.go index 550696a31..28d89e864 100644 --- a/pilosa.go +++ b/pilosa.go @@ -166,7 +166,6 @@ func stringSlicesAreEqual(a, b []string) bool { func AddressWithDefaults(addr string) (*URI, error) { if addr == "" { return defaultURI(), nil - } else { - return NewURIFromAddress(addr) } + return NewURIFromAddress(addr) } diff --git a/pql/ast.go b/pql/ast.go index 8b23c708b..d78ad2828 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -259,7 +259,7 @@ func (c *Call) FieldArg() (string, error) { return arg, nil } } - return "", fmt.Errorf("No field argument specified") + return "", fmt.Errorf("no field argument specified") } func IsReservedArg(name string) bool { diff --git a/pql/pql.peg b/pql/pql.peg index 66c1d3265..8127ecc83 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -13,6 +13,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} + / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 8df2fcb72..8d960f95c 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -62,9 +62,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 - rulePegText ruleAction14 ruleAction15 + rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -99,6 +99,8 @@ const ( ruleAction47 ruleAction48 ruleAction49 + ruleAction50 + ruleAction51 ) var rul3s = [...]string{ @@ -149,9 +151,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", - "PegText", "Action14", "Action15", + "PegText", "Action16", "Action17", "Action18", @@ -186,6 +188,8 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", + "Action50", + "Action51", } type token32 struct { @@ -302,7 +306,7 @@ type PQL struct { Buffer string buffer []rune - rules [84]func() bool + rules [86]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -423,77 +427,81 @@ func (p *PQL) Execute() { case ruleAction13: p.endCall() case ruleAction14: - p.startCall(buffer[begin:end]) + p.startCall("Rows") case ruleAction15: p.endCall() case ruleAction16: - p.addBTWN() + p.startCall(buffer[begin:end]) case ruleAction17: - p.addLTE() + p.endCall() case ruleAction18: - p.addGTE() + p.addBTWN() case ruleAction19: - p.addEQ() + p.addLTE() case ruleAction20: - p.addNEQ() + p.addGTE() case ruleAction21: - p.addLT() + p.addEQ() case ruleAction22: - p.addGT() + p.addNEQ() case ruleAction23: - p.startConditional() + p.addLT() case ruleAction24: - p.endConditional() + p.addGT() case ruleAction25: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction26: - p.condAdd(buffer[begin:end]) + p.endConditional() case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.startList() + p.condAdd(buffer[begin:end]) case ruleAction29: - p.endList() + p.condAdd(buffer[begin:end]) case ruleAction30: - p.addVal(nil) + p.startList() case ruleAction31: - p.addVal(true) + p.endList() case ruleAction32: - p.addVal(false) + p.addVal(nil) case ruleAction33: - p.addVal(buffer[begin:end]) + p.addVal(true) case ruleAction34: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction35: - p.addNumVal(buffer[begin:end]) - case ruleAction36: - p.startCall(buffer[begin:end]) - case ruleAction37: - p.addVal(p.endCall()) - case ruleAction38: p.addVal(buffer[begin:end]) + case ruleAction36: + p.addNumVal(buffer[begin:end]) + case ruleAction37: + p.addNumVal(buffer[begin:end]) + case ruleAction38: + p.startCall(buffer[begin:end]) case ruleAction39: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(p.endCall()) case ruleAction40: p.addVal(buffer[begin:end]) case ruleAction41: - p.addField(buffer[begin:end]) + s, _ := strconv.Unquote(buffer[begin:end]) + p.addVal(s) case ruleAction42: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction43: - p.addPosNum("_col", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction44: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction45: - p.addPosStr("_col", buffer[begin:end]) + p.addPosNum("_col", buffer[begin:end]) case ruleAction46: - p.addPosNum("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction47: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction49: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -606,7 +614,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ( Action14 open allargs comma? close Action15))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'o' 'w' 's' Action14 open posfield (comma allargs)? close Action15) / ( Action16 open allargs comma? close Action17))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -655,7 +663,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction49, position) + add(ruleAction51, position) } add(ruletimestamp, position12) } @@ -741,7 +749,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction46, position) + add(ruleAction48, position) } goto l19 l20: @@ -762,7 +770,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction47, position) + add(ruleAction49, position) } goto l19 l23: @@ -783,7 +791,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction50, position) } } l19: @@ -1069,38 +1077,85 @@ func (p *PQL) Init() { goto l7 l41: position, tokenIndex = position7, tokenIndex7 - { - position46 := position - if !_rules[ruleIDENT]() { - goto l5 - } - add(rulePegText, position46) + if buffer[position] != rune('R') { + goto l46 } + position++ + if buffer[position] != rune('o') { + goto l46 + } + position++ + if buffer[position] != rune('w') { + goto l46 + } + position++ + if buffer[position] != rune('s') { + goto l46 + } + position++ { add(ruleAction14, position) } if !_rules[ruleopen]() { - goto l5 + goto l46 } - if !_rules[ruleallargs]() { - goto l5 + if !_rules[ruleposfield]() { + goto l46 } { position48, tokenIndex48 := position, tokenIndex if !_rules[rulecomma]() { goto l48 } + if !_rules[ruleallargs]() { + goto l48 + } goto l49 l48: position, tokenIndex = position48, tokenIndex48 } l49: if !_rules[ruleclose]() { - goto l5 + goto l46 } { add(ruleAction15, position) } + goto l7 + l46: + position, tokenIndex = position7, tokenIndex7 + { + position51 := position + if !_rules[ruleIDENT]() { + goto l5 + } + add(rulePegText, position51) + } + { + add(ruleAction16, position) + } + if !_rules[ruleopen]() { + goto l5 + } + if !_rules[ruleallargs]() { + goto l5 + } + { + position53, tokenIndex53 := position, tokenIndex + if !_rules[rulecomma]() { + goto l53 + } + goto l54 + l53: + position, tokenIndex = position53, tokenIndex53 + } + l54: + if !_rules[ruleclose]() { + goto l5 + } + { + add(ruleAction17, position) + } } l7: add(ruleCall, position6) @@ -1112,520 +1167,474 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position51, tokenIndex51 := position, tokenIndex + position56, tokenIndex56 := position, tokenIndex { - position52 := position + position57 := position { - position53, tokenIndex53 := position, tokenIndex + position58, tokenIndex58 := position, tokenIndex if !_rules[ruleCall]() { - goto l54 - } - l55: - { - position56, tokenIndex56 := position, tokenIndex - if !_rules[rulecomma]() { - goto l56 - } - if !_rules[ruleCall]() { - goto l56 - } - goto l55 - l56: - position, tokenIndex = position56, tokenIndex56 - } - { - position57, tokenIndex57 := position, tokenIndex - if !_rules[rulecomma]() { - goto l57 - } - if !_rules[ruleargs]() { - goto l57 - } - goto l58 - l57: - position, tokenIndex = position57, tokenIndex57 - } - l58: - goto l53 - l54: - position, tokenIndex = position53, tokenIndex53 - if !_rules[ruleargs]() { goto l59 } - goto l53 + l60: + { + position61, tokenIndex61 := position, tokenIndex + if !_rules[rulecomma]() { + goto l61 + } + if !_rules[ruleCall]() { + goto l61 + } + goto l60 + l61: + position, tokenIndex = position61, tokenIndex61 + } + { + position62, tokenIndex62 := position, tokenIndex + if !_rules[rulecomma]() { + goto l62 + } + if !_rules[ruleargs]() { + goto l62 + } + goto l63 + l62: + position, tokenIndex = position62, tokenIndex62 + } + l63: + goto l58 l59: - position, tokenIndex = position53, tokenIndex53 + position, tokenIndex = position58, tokenIndex58 + if !_rules[ruleargs]() { + goto l64 + } + goto l58 + l64: + position, tokenIndex = position58, tokenIndex58 if !_rules[rulesp]() { - goto l51 + goto l56 } } - l53: - add(ruleallargs, position52) + l58: + add(ruleallargs, position57) } return true - l51: - position, tokenIndex = position51, tokenIndex51 + l56: + position, tokenIndex = position56, tokenIndex56 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position60, tokenIndex60 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex { - position61 := position + position66 := position if !_rules[rulearg]() { - goto l60 + goto l65 } { - position62, tokenIndex62 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex if !_rules[rulecomma]() { - goto l62 + goto l67 } if !_rules[ruleargs]() { - goto l62 + goto l67 } - goto l63 - l62: - position, tokenIndex = position62, tokenIndex62 + goto l68 + l67: + position, tokenIndex = position67, tokenIndex67 } - l63: + l68: if !_rules[rulesp]() { - goto l60 + goto l65 } - add(ruleargs, position61) + add(ruleargs, position66) } return true - l60: - position, tokenIndex = position60, tokenIndex60 + l65: + position, tokenIndex = position65, tokenIndex65 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value) / conditional)> */ func() bool { - position64, tokenIndex64 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex { - position65 := position + position70 := position { - position66, tokenIndex66 := position, tokenIndex + position71, tokenIndex71 := position, tokenIndex if !_rules[rulefield]() { - goto l67 + goto l72 } if !_rules[rulesp]() { - goto l67 + goto l72 } if buffer[position] != rune('=') { - goto l67 + goto l72 } position++ if !_rules[rulesp]() { - goto l67 + goto l72 } if !_rules[rulevalue]() { - goto l67 + goto l72 } - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 + goto l71 + l72: + position, tokenIndex = position71, tokenIndex71 if !_rules[rulefield]() { - goto l68 + goto l73 } if !_rules[rulesp]() { - goto l68 + goto l73 } { - position69 := position + position74 := position { - position70, tokenIndex70 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex if buffer[position] != rune('>') { - goto l71 + goto l76 } position++ if buffer[position] != rune('<') { - goto l71 - } - position++ - { - add(ruleAction16, position) - } - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('<') { - goto l73 - } - position++ - if buffer[position] != rune('=') { - goto l73 - } - position++ - { - add(ruleAction17, position) - } - goto l70 - l73: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('>') { - goto l75 - } - position++ - if buffer[position] != rune('=') { - goto l75 + goto l76 } position++ { add(ruleAction18, position) } - goto l70 - l75: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('=') { - goto l77 + goto l75 + l76: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('<') { + goto l78 } position++ if buffer[position] != rune('=') { - goto l77 + goto l78 } position++ { add(ruleAction19, position) } - goto l70 - l77: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('!') { - goto l79 + goto l75 + l78: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('>') { + goto l80 } position++ if buffer[position] != rune('=') { - goto l79 + goto l80 } position++ { add(ruleAction20, position) } - goto l70 - l79: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('<') { - goto l81 + goto l75 + l80: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('=') { + goto l82 + } + position++ + if buffer[position] != rune('=') { + goto l82 } position++ { add(ruleAction21, position) } - goto l70 - l81: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('>') { - goto l68 + goto l75 + l82: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('!') { + goto l84 + } + position++ + if buffer[position] != rune('=') { + goto l84 } position++ { add(ruleAction22, position) } + goto l75 + l84: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('<') { + goto l86 + } + position++ + { + add(ruleAction23, position) + } + goto l75 + l86: + position, tokenIndex = position75, tokenIndex75 + if buffer[position] != rune('>') { + goto l73 + } + position++ + { + add(ruleAction24, position) + } } - l70: - add(ruleCOND, position69) + l75: + add(ruleCOND, position74) } if !_rules[rulesp]() { - goto l68 + goto l73 } if !_rules[rulevalue]() { - goto l68 + goto l73 } - goto l66 - l68: - position, tokenIndex = position66, tokenIndex66 + goto l71 + l73: + position, tokenIndex = position71, tokenIndex71 { - position84 := position + position89 := position { - add(ruleAction23, position) + add(ruleAction25, position) } if !_rules[rulecondint]() { - goto l64 + goto l69 } if !_rules[rulecondLT]() { - goto l64 + goto l69 } { - position86 := position + position91 := position { - position87 := position + position92 := position if !_rules[rulefieldExpr]() { - goto l64 + goto l69 } - add(rulePegText, position87) + add(rulePegText, position92) } if !_rules[rulesp]() { - goto l64 + goto l69 } { - add(ruleAction27, position) + add(ruleAction29, position) } - add(rulecondfield, position86) + add(rulecondfield, position91) } if !_rules[rulecondLT]() { - goto l64 + goto l69 } if !_rules[rulecondint]() { - goto l64 + goto l69 } { - add(ruleAction24, position) + add(ruleAction26, position) } - add(ruleconditional, position84) + add(ruleconditional, position89) } } - l66: - add(rulearg, position65) + l71: + add(rulearg, position70) } return true - l64: - position, tokenIndex = position64, tokenIndex64 + l69: + position, tokenIndex = position69, tokenIndex69 return false }, - /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ + /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ nil, - /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ + /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ func() bool { - position92, tokenIndex92 := position, tokenIndex + position97, tokenIndex97 := position, tokenIndex { - position93 := position + position98 := position { - position94 := position + position99 := position { - position95, tokenIndex95 := position, tokenIndex + position100, tokenIndex100 := position, tokenIndex { - position97, tokenIndex97 := position, tokenIndex + position102, tokenIndex102 := position, tokenIndex if buffer[position] != rune('-') { - goto l97 + goto l102 } position++ - goto l98 - l97: - position, tokenIndex = position97, tokenIndex97 + goto l103 + l102: + position, tokenIndex = position102, tokenIndex102 } - l98: + l103: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l96 + goto l101 } position++ - l99: + l104: { - position100, tokenIndex100 := position, tokenIndex + position105, tokenIndex105 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l100 + goto l105 } position++ - goto l99 - l100: - position, tokenIndex = position100, tokenIndex100 + goto l104 + l105: + position, tokenIndex = position105, tokenIndex105 } - goto l95 - l96: - position, tokenIndex = position95, tokenIndex95 + goto l100 + l101: + position, tokenIndex = position100, tokenIndex100 if buffer[position] != rune('0') { - goto l92 + goto l97 } position++ } - l95: - add(rulePegText, position94) + l100: + add(rulePegText, position99) } if !_rules[rulesp]() { - goto l92 + goto l97 } { - add(ruleAction25, position) + add(ruleAction27, position) } - add(rulecondint, position93) + add(rulecondint, position98) } return true - l92: - position, tokenIndex = position92, tokenIndex92 + l97: + position, tokenIndex = position97, tokenIndex97 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position102, tokenIndex102 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex { - position103 := position + position108 := position { - position104 := position + position109 := position { - position105, tokenIndex105 := position, tokenIndex + position110, tokenIndex110 := position, tokenIndex if buffer[position] != rune('<') { - goto l106 + goto l111 } position++ if buffer[position] != rune('=') { - goto l106 + goto l111 } position++ - goto l105 - l106: - position, tokenIndex = position105, tokenIndex105 + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 if buffer[position] != rune('<') { - goto l102 + goto l107 } position++ } - l105: - add(rulePegText, position104) + l110: + add(rulePegText, position109) } if !_rules[rulesp]() { - goto l102 + goto l107 } { - add(ruleAction26, position) + add(ruleAction28, position) } - add(rulecondLT, position103) + add(rulecondLT, position108) } return true - l102: - position, tokenIndex = position102, tokenIndex102 + l107: + position, tokenIndex = position107, tokenIndex107 return false }, - /* 9 condfield <- <( sp Action27)> */ + /* 9 condfield <- <( sp Action29)> */ nil, - /* 10 value <- <(item / (lbrack Action28 list rbrack Action29))> */ + /* 10 value <- <(item / (lbrack Action30 list rbrack Action31))> */ func() bool { - position109, tokenIndex109 := position, tokenIndex + position114, tokenIndex114 := position, tokenIndex { - position110 := position + position115 := position { - position111, tokenIndex111 := position, tokenIndex + position116, tokenIndex116 := position, tokenIndex if !_rules[ruleitem]() { - goto l112 + goto l117 } - goto l111 - l112: - position, tokenIndex = position111, tokenIndex111 + goto l116 + l117: + position, tokenIndex = position116, tokenIndex116 { - position113 := position + position118 := position if buffer[position] != rune('[') { - goto l109 + goto l114 } position++ if !_rules[rulesp]() { - goto l109 + goto l114 } - add(rulelbrack, position113) - } - { - add(ruleAction28, position) - } - if !_rules[rulelist]() { - goto l109 - } - { - position115 := position - if !_rules[rulesp]() { - goto l109 - } - if buffer[position] != rune(']') { - goto l109 - } - position++ - if !_rules[rulesp]() { - goto l109 - } - add(rulerbrack, position115) - } - { - add(ruleAction29, position) - } - } - l111: - add(rulevalue, position110) - } - return true - l109: - position, tokenIndex = position109, tokenIndex109 - return false - }, - /* 11 list <- <(item (comma list)?)> */ - func() bool { - position117, tokenIndex117 := position, tokenIndex - { - position118 := position - if !_rules[ruleitem]() { - goto l117 - } - { - position119, tokenIndex119 := position, tokenIndex - if !_rules[rulecomma]() { - goto l119 - } - if !_rules[rulelist]() { - goto l119 - } - goto l120 - l119: - position, tokenIndex = position119, tokenIndex119 - } - l120: - add(rulelist, position118) - } - return true - l117: - position, tokenIndex = position117, tokenIndex117 - return false - }, - /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (timestampfmt Action33) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action34) / (<('-'? '.' [0-9]+)> Action35) / ( Action36 open allargs comma? close Action37) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action38) / (<('"' doublequotedstring '"')> Action39) / ('\'' '\'' Action40))> */ - func() bool { - position121, tokenIndex121 := position, tokenIndex - { - position122 := position - { - position123, tokenIndex123 := position, tokenIndex - if buffer[position] != rune('n') { - goto l124 - } - position++ - if buffer[position] != rune('u') { - goto l124 - } - position++ - if buffer[position] != rune('l') { - goto l124 - } - position++ - if buffer[position] != rune('l') { - goto l124 - } - position++ - { - position125, tokenIndex125 := position, tokenIndex - { - position126, tokenIndex126 := position, tokenIndex - if !_rules[rulecomma]() { - goto l127 - } - goto l126 - l127: - position, tokenIndex = position126, tokenIndex126 - if !_rules[rulesp]() { - goto l124 - } - if !_rules[ruleclose]() { - goto l124 - } - } - l126: - position, tokenIndex = position125, tokenIndex125 + add(rulelbrack, position118) } { add(ruleAction30, position) } - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('t') { - goto l129 + if !_rules[rulelist]() { + goto l114 } - position++ - if buffer[position] != rune('r') { + { + position120 := position + if !_rules[rulesp]() { + goto l114 + } + if buffer[position] != rune(']') { + goto l114 + } + position++ + if !_rules[rulesp]() { + goto l114 + } + add(rulerbrack, position120) + } + { + add(ruleAction31, position) + } + } + l116: + add(rulevalue, position115) + } + return true + l114: + position, tokenIndex = position114, tokenIndex114 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position122, tokenIndex122 := position, tokenIndex + { + position123 := position + if !_rules[ruleitem]() { + goto l122 + } + { + position124, tokenIndex124 := position, tokenIndex + if !_rules[rulecomma]() { + goto l124 + } + if !_rules[rulelist]() { + goto l124 + } + goto l125 + l124: + position, tokenIndex = position124, tokenIndex124 + } + l125: + add(rulelist, position123) + } + return true + l122: + position, tokenIndex = position122, tokenIndex122 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (timestampfmt Action35) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action36) / (<('-'? '.' [0-9]+)> Action37) / ( Action38 open allargs comma? close Action39) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action40) / (<('"' doublequotedstring '"')> Action41) / ('\'' '\'' Action42))> */ + func() bool { + position126, tokenIndex126 := position, tokenIndex + { + position127 := position + { + position128, tokenIndex128 := position, tokenIndex + if buffer[position] != rune('n') { goto l129 } position++ @@ -1633,7 +1642,11 @@ func (p *PQL) Init() { goto l129 } position++ - if buffer[position] != rune('e') { + if buffer[position] != rune('l') { + goto l129 + } + position++ + if buffer[position] != rune('l') { goto l129 } position++ @@ -1658,24 +1671,20 @@ func (p *PQL) Init() { position, tokenIndex = position130, tokenIndex130 } { - add(ruleAction31, position) + add(ruleAction32, position) } - goto l123 + goto l128 l129: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('f') { + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('t') { goto l134 } position++ - if buffer[position] != rune('a') { + if buffer[position] != rune('r') { goto l134 } position++ - if buffer[position] != rune('l') { - goto l134 - } - position++ - if buffer[position] != rune('s') { + if buffer[position] != rune('u') { goto l134 } position++ @@ -1703,898 +1712,944 @@ func (p *PQL) Init() { l136: position, tokenIndex = position135, tokenIndex135 } - { - add(ruleAction32, position) - } - goto l123 - l134: - position, tokenIndex = position123, tokenIndex123 - if !_rules[ruletimestampfmt]() { - goto l139 - } { add(ruleAction33, position) } - goto l123 - l139: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l134: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('f') { + goto l139 + } + position++ + if buffer[position] != rune('a') { + goto l139 + } + position++ + if buffer[position] != rune('l') { + goto l139 + } + position++ + if buffer[position] != rune('s') { + goto l139 + } + position++ + if buffer[position] != rune('e') { + goto l139 + } + position++ { - position142 := position + position140, tokenIndex140 := position, tokenIndex { - position143, tokenIndex143 := position, tokenIndex - if buffer[position] != rune('-') { - goto l143 + position141, tokenIndex141 := position, tokenIndex + if !_rules[rulecomma]() { + goto l142 } - position++ - goto l144 - l143: - position, tokenIndex = position143, tokenIndex143 - } - l144: - if c := buffer[position]; c < rune('0') || c > rune('9') { goto l141 - } - position++ - l145: - { - position146, tokenIndex146 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l146 + l142: + position, tokenIndex = position141, tokenIndex141 + if !_rules[rulesp]() { + goto l139 } - position++ - goto l145 - l146: - position, tokenIndex = position146, tokenIndex146 - } - { - position147, tokenIndex147 := position, tokenIndex - if buffer[position] != rune('.') { - goto l147 + if !_rules[ruleclose]() { + goto l139 } - position++ - l149: - { - position150, tokenIndex150 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l150 - } - position++ - goto l149 - l150: - position, tokenIndex = position150, tokenIndex150 - } - goto l148 - l147: - position, tokenIndex = position147, tokenIndex147 } - l148: - add(rulePegText, position142) + l141: + position, tokenIndex = position140, tokenIndex140 } { add(ruleAction34, position) } - goto l123 - l141: - position, tokenIndex = position123, tokenIndex123 - { - position153 := position - { - position154, tokenIndex154 := position, tokenIndex - if buffer[position] != rune('-') { - goto l154 - } - position++ - goto l155 - l154: - position, tokenIndex = position154, tokenIndex154 - } - l155: - if buffer[position] != rune('.') { - goto l152 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 - } - position++ - l156: - { - position157, tokenIndex157 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l157 - } - position++ - goto l156 - l157: - position, tokenIndex = position157, tokenIndex157 - } - add(rulePegText, position153) + goto l128 + l139: + position, tokenIndex = position128, tokenIndex128 + if !_rules[ruletimestampfmt]() { + goto l144 } { add(ruleAction35, position) } - goto l123 - l152: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l144: + position, tokenIndex = position128, tokenIndex128 { - position160 := position - if !_rules[ruleIDENT]() { - goto l159 + position147 := position + { + position148, tokenIndex148 := position, tokenIndex + if buffer[position] != rune('-') { + goto l148 + } + position++ + goto l149 + l148: + position, tokenIndex = position148, tokenIndex148 } - add(rulePegText, position160) + l149: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l146 + } + position++ + l150: + { + position151, tokenIndex151 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l151 + } + position++ + goto l150 + l151: + position, tokenIndex = position151, tokenIndex151 + } + { + position152, tokenIndex152 := position, tokenIndex + if buffer[position] != rune('.') { + goto l152 + } + position++ + l154: + { + position155, tokenIndex155 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l155 + } + position++ + goto l154 + l155: + position, tokenIndex = position155, tokenIndex155 + } + goto l153 + l152: + position, tokenIndex = position152, tokenIndex152 + } + l153: + add(rulePegText, position147) } { add(ruleAction36, position) } - if !_rules[ruleopen]() { - goto l159 - } - if !_rules[ruleallargs]() { - goto l159 - } + goto l128 + l146: + position, tokenIndex = position128, tokenIndex128 { - position162, tokenIndex162 := position, tokenIndex - if !_rules[rulecomma]() { - goto l162 + position158 := position + { + position159, tokenIndex159 := position, tokenIndex + if buffer[position] != rune('-') { + goto l159 + } + position++ + goto l160 + l159: + position, tokenIndex = position159, tokenIndex159 } - goto l163 - l162: - position, tokenIndex = position162, tokenIndex162 - } - l163: - if !_rules[ruleclose]() { - goto l159 + l160: + if buffer[position] != rune('.') { + goto l157 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l157 + } + position++ + l161: + { + position162, tokenIndex162 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l162 + } + position++ + goto l161 + l162: + position, tokenIndex = position162, tokenIndex162 + } + add(rulePegText, position158) } { add(ruleAction37, position) } - goto l123 - l159: - position, tokenIndex = position123, tokenIndex123 + goto l128 + l157: + position, tokenIndex = position128, tokenIndex128 { - position166 := position - { - position169, tokenIndex169 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l170 - } - position++ - goto l169 - l170: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l171 - } - position++ - goto l169 - l171: - position, tokenIndex = position169, tokenIndex169 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l172 - } - position++ - goto l169 - l172: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune('-') { - goto l173 - } - position++ - goto l169 - l173: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune('_') { - goto l174 - } - position++ - goto l169 - l174: - position, tokenIndex = position169, tokenIndex169 - if buffer[position] != rune(':') { - goto l165 - } - position++ + position165 := position + if !_rules[ruleIDENT]() { + goto l164 } - l169: - l167: - { - position168, tokenIndex168 := position, tokenIndex - { - position175, tokenIndex175 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l176 - } - position++ - goto l175 - l176: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l177 - } - position++ - goto l175 - l177: - position, tokenIndex = position175, tokenIndex175 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l178 - } - position++ - goto l175 - l178: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('-') { - goto l179 - } - position++ - goto l175 - l179: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune('_') { - goto l180 - } - position++ - goto l175 - l180: - position, tokenIndex = position175, tokenIndex175 - if buffer[position] != rune(':') { - goto l168 - } - position++ - } - l175: - goto l167 - l168: - position, tokenIndex = position168, tokenIndex168 - } - add(rulePegText, position166) + add(rulePegText, position165) } { add(ruleAction38, position) } - goto l123 - l165: - position, tokenIndex = position123, tokenIndex123 + if !_rules[ruleopen]() { + goto l164 + } + if !_rules[ruleallargs]() { + goto l164 + } { - position183 := position - if buffer[position] != rune('"') { - goto l182 + position167, tokenIndex167 := position, tokenIndex + if !_rules[rulecomma]() { + goto l167 } - position++ - if !_rules[ruledoublequotedstring]() { - goto l182 - } - if buffer[position] != rune('"') { - goto l182 - } - position++ - add(rulePegText, position183) + goto l168 + l167: + position, tokenIndex = position167, tokenIndex167 + } + l168: + if !_rules[ruleclose]() { + goto l164 } { add(ruleAction39, position) } - goto l123 - l182: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('\'') { - goto l121 - } - position++ + goto l128 + l164: + position, tokenIndex = position128, tokenIndex128 { - position185 := position - if !_rules[rulesinglequotedstring]() { - goto l121 + position171 := position + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l175 + } + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l176 + } + position++ + goto l174 + l176: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l177 + } + position++ + goto l174 + l177: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('-') { + goto l178 + } + position++ + goto l174 + l178: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('_') { + goto l179 + } + position++ + goto l174 + l179: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune(':') { + goto l170 + } + position++ } - add(rulePegText, position185) + l174: + l172: + { + position173, tokenIndex173 := position, tokenIndex + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l173 + } + position++ + } + l180: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + add(rulePegText, position171) } - if buffer[position] != rune('\'') { - goto l121 - } - position++ { add(ruleAction40, position) } + goto l128 + l170: + position, tokenIndex = position128, tokenIndex128 + { + position188 := position + if buffer[position] != rune('"') { + goto l187 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l187 + } + if buffer[position] != rune('"') { + goto l187 + } + position++ + add(rulePegText, position188) + } + { + add(ruleAction41, position) + } + goto l128 + l187: + position, tokenIndex = position128, tokenIndex128 + if buffer[position] != rune('\'') { + goto l126 + } + position++ + { + position190 := position + if !_rules[rulesinglequotedstring]() { + goto l126 + } + add(rulePegText, position190) + } + if buffer[position] != rune('\'') { + goto l126 + } + position++ + { + add(ruleAction42, position) + } } - l123: - add(ruleitem, position122) + l128: + add(ruleitem, position127) } return true - l121: - position, tokenIndex = position121, tokenIndex121 + l126: + position, tokenIndex = position126, tokenIndex126 return false }, /* 13 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position188 := position - l189: + position193 := position + l194: { - position190, tokenIndex190 := position, tokenIndex + position195, tokenIndex195 := position, tokenIndex { - position191, tokenIndex191 := position, tokenIndex + position196, tokenIndex196 := position, tokenIndex if buffer[position] != rune('\\') { - goto l192 + goto l197 } position++ if buffer[position] != rune('"') { - goto l192 + goto l197 } position++ - goto l191 - l192: - position, tokenIndex = position191, tokenIndex191 + goto l196 + l197: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l193 + goto l198 } position++ if buffer[position] != rune('\\') { - goto l193 + goto l198 } position++ - goto l191 - l193: - position, tokenIndex = position191, tokenIndex191 + goto l196 + l198: + position, tokenIndex = position196, tokenIndex196 { - position194, tokenIndex194 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex if buffer[position] != rune('"') { - goto l194 + goto l199 } position++ - goto l190 - l194: - position, tokenIndex = position194, tokenIndex194 + goto l195 + l199: + position, tokenIndex = position199, tokenIndex199 } if !matchDot() { - goto l190 + goto l195 } } - l191: - goto l189 - l190: - position, tokenIndex = position190, tokenIndex190 + l196: + goto l194 + l195: + position, tokenIndex = position195, tokenIndex195 } - add(ruledoublequotedstring, position188) + add(ruledoublequotedstring, position193) } return true }, /* 14 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position196 := position - l197: + position201 := position + l202: { - position198, tokenIndex198 := position, tokenIndex + position203, tokenIndex203 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position204, tokenIndex204 := position, tokenIndex if buffer[position] != rune('\\') { - goto l200 + goto l205 } position++ if buffer[position] != rune('\'') { - goto l200 + goto l205 } position++ - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l204 + l205: + position, tokenIndex = position204, tokenIndex204 if buffer[position] != rune('\\') { - goto l201 + goto l206 } position++ if buffer[position] != rune('\\') { - goto l201 + goto l206 } position++ - goto l199 - l201: - position, tokenIndex = position199, tokenIndex199 + goto l204 + l206: + position, tokenIndex = position204, tokenIndex204 { - position202, tokenIndex202 := position, tokenIndex + position207, tokenIndex207 := position, tokenIndex if buffer[position] != rune('\'') { - goto l202 + goto l207 } position++ - goto l198 - l202: - position, tokenIndex = position202, tokenIndex202 + goto l203 + l207: + position, tokenIndex = position207, tokenIndex207 } if !matchDot() { - goto l198 + goto l203 } } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l204: + goto l202 + l203: + position, tokenIndex = position203, tokenIndex203 } - add(rulesinglequotedstring, position196) + add(rulesinglequotedstring, position201) } return true }, /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position203, tokenIndex203 := position, tokenIndex + position208, tokenIndex208 := position, tokenIndex { - position204 := position + position209 := position { - position205, tokenIndex205 := position, tokenIndex + position210, tokenIndex210 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l206 + goto l211 } position++ - goto l205 - l206: - position, tokenIndex = position205, tokenIndex205 + goto l210 + l211: + position, tokenIndex = position210, tokenIndex210 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l203 + goto l208 } position++ } - l205: - l207: + l210: + l212: { - position208, tokenIndex208 := position, tokenIndex + position213, tokenIndex213 := position, tokenIndex { - position209, tokenIndex209 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l210 + goto l215 } position++ - goto l209 - l210: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l215: + position, tokenIndex = position214, tokenIndex214 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l211 + goto l216 } position++ - goto l209 - l211: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l216: + position, tokenIndex = position214, tokenIndex214 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l212 + goto l217 } position++ - goto l209 - l212: - position, tokenIndex = position209, tokenIndex209 + goto l214 + l217: + position, tokenIndex = position214, tokenIndex214 if buffer[position] != rune('_') { + goto l218 + } + position++ + goto l214 + l218: + position, tokenIndex = position214, tokenIndex214 + if buffer[position] != rune('-') { goto l213 } position++ - goto l209 - l213: - position, tokenIndex = position209, tokenIndex209 - if buffer[position] != rune('-') { - goto l208 - } - position++ } - l209: - goto l207 - l208: - position, tokenIndex = position208, tokenIndex208 + l214: + goto l212 + l213: + position, tokenIndex = position213, tokenIndex213 } - add(rulefieldExpr, position204) + add(rulefieldExpr, position209) } return true - l203: - position, tokenIndex = position203, tokenIndex203 + l208: + position, tokenIndex = position208, tokenIndex208 return false }, - /* 16 field <- <(<(fieldExpr / reserved)> Action41)> */ + /* 16 field <- <(<(fieldExpr / reserved)> Action43)> */ func() bool { - position214, tokenIndex214 := position, tokenIndex + position219, tokenIndex219 := position, tokenIndex { - position215 := position + position220 := position { - position216 := position + position221 := position { - position217, tokenIndex217 := position, tokenIndex + position222, tokenIndex222 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l218 + goto l223 } - goto l217 - l218: - position, tokenIndex = position217, tokenIndex217 + goto l222 + l223: + position, tokenIndex = position222, tokenIndex222 { - position219 := position + position224 := position { - position220, tokenIndex220 := position, tokenIndex + position225, tokenIndex225 := position, tokenIndex if buffer[position] != rune('_') { - goto l221 + goto l226 } position++ if buffer[position] != rune('r') { - goto l221 + goto l226 } position++ if buffer[position] != rune('o') { - goto l221 + goto l226 } position++ if buffer[position] != rune('w') { - goto l221 + goto l226 } position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l226: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l222 + goto l227 } position++ if buffer[position] != rune('c') { - goto l222 + goto l227 } position++ if buffer[position] != rune('o') { - goto l222 + goto l227 } position++ if buffer[position] != rune('l') { - goto l222 + goto l227 } position++ - goto l220 - l222: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l227: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l223 + goto l228 } position++ if buffer[position] != rune('s') { - goto l223 + goto l228 } position++ if buffer[position] != rune('t') { - goto l223 + goto l228 } position++ if buffer[position] != rune('a') { - goto l223 + goto l228 } position++ if buffer[position] != rune('r') { - goto l223 + goto l228 } position++ if buffer[position] != rune('t') { - goto l223 + goto l228 } position++ - goto l220 - l223: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l228: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l224 + goto l229 } position++ if buffer[position] != rune('e') { - goto l224 + goto l229 } position++ if buffer[position] != rune('n') { - goto l224 + goto l229 } position++ if buffer[position] != rune('d') { - goto l224 + goto l229 } position++ - goto l220 - l224: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l229: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l225 + goto l230 } position++ if buffer[position] != rune('t') { - goto l225 + goto l230 } position++ if buffer[position] != rune('i') { - goto l225 + goto l230 } position++ if buffer[position] != rune('m') { - goto l225 + goto l230 } position++ if buffer[position] != rune('e') { - goto l225 + goto l230 } position++ if buffer[position] != rune('s') { - goto l225 + goto l230 } position++ if buffer[position] != rune('t') { - goto l225 + goto l230 } position++ if buffer[position] != rune('a') { - goto l225 + goto l230 } position++ if buffer[position] != rune('m') { - goto l225 + goto l230 } position++ if buffer[position] != rune('p') { - goto l225 + goto l230 } position++ - goto l220 - l225: - position, tokenIndex = position220, tokenIndex220 + goto l225 + l230: + position, tokenIndex = position225, tokenIndex225 if buffer[position] != rune('_') { - goto l214 + goto l219 } position++ if buffer[position] != rune('f') { - goto l214 + goto l219 } position++ if buffer[position] != rune('i') { - goto l214 + goto l219 } position++ if buffer[position] != rune('e') { - goto l214 + goto l219 } position++ if buffer[position] != rune('l') { - goto l214 + goto l219 } position++ if buffer[position] != rune('d') { - goto l214 + goto l219 } position++ } - l220: - add(rulereserved, position219) + l225: + add(rulereserved, position224) } } - l217: - add(rulePegText, position216) + l222: + add(rulePegText, position221) } { - add(ruleAction41, position) + add(ruleAction43, position) } - add(rulefield, position215) + add(rulefield, position220) } return true - l214: - position, tokenIndex = position214, tokenIndex214 + l219: + position, tokenIndex = position219, tokenIndex219 return false }, /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 18 posfield <- <( Action42)> */ + /* 18 posfield <- <( Action44)> */ func() bool { - position228, tokenIndex228 := position, tokenIndex + position233, tokenIndex233 := position, tokenIndex { - position229 := position + position234 := position { - position230 := position + position235 := position if !_rules[rulefieldExpr]() { - goto l228 + goto l233 } - add(rulePegText, position230) + add(rulePegText, position235) } { - add(ruleAction42, position) + add(ruleAction44, position) } - add(ruleposfield, position229) + add(ruleposfield, position234) } return true - l228: - position, tokenIndex = position228, tokenIndex228 + l233: + position, tokenIndex = position233, tokenIndex233 return false }, /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position232, tokenIndex232 := position, tokenIndex + position237, tokenIndex237 := position, tokenIndex { - position233 := position + position238 := position { - position234, tokenIndex234 := position, tokenIndex + position239, tokenIndex239 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l235 + goto l240 } position++ - l236: + l241: { - position237, tokenIndex237 := position, tokenIndex + position242, tokenIndex242 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l237 + goto l242 } position++ - goto l236 - l237: - position, tokenIndex = position237, tokenIndex237 + goto l241 + l242: + position, tokenIndex = position242, tokenIndex242 } - goto l234 - l235: - position, tokenIndex = position234, tokenIndex234 + goto l239 + l240: + position, tokenIndex = position239, tokenIndex239 if buffer[position] != rune('0') { - goto l232 + goto l237 } position++ } - l234: - add(ruleuint, position233) + l239: + add(ruleuint, position238) } return true - l232: - position, tokenIndex = position232, tokenIndex232 + l237: + position, tokenIndex = position237, tokenIndex237 return false }, - /* 20 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ + /* 20 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ func() bool { - position238, tokenIndex238 := position, tokenIndex + position243, tokenIndex243 := position, tokenIndex { - position239 := position + position244 := position { - position240, tokenIndex240 := position, tokenIndex - { - position242 := position - if !_rules[ruleuint]() { - goto l241 - } - add(rulePegText, position242) - } - { - add(ruleAction43, position) - } - goto l240 - l241: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('\'') { - goto l244 - } - position++ - { - position245 := position - if !_rules[rulesinglequotedstring]() { - goto l244 - } - add(rulePegText, position245) - } - if buffer[position] != rune('\'') { - goto l244 - } - position++ - { - add(ruleAction44, position) - } - goto l240 - l244: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('"') { - goto l238 - } - position++ + position245, tokenIndex245 := position, tokenIndex { position247 := position - if !_rules[ruledoublequotedstring]() { - goto l238 + if !_rules[ruleuint]() { + goto l246 } add(rulePegText, position247) } - if buffer[position] != rune('"') { - goto l238 - } - position++ { add(ruleAction45, position) } + goto l245 + l246: + position, tokenIndex = position245, tokenIndex245 + if buffer[position] != rune('\'') { + goto l249 + } + position++ + { + position250 := position + if !_rules[rulesinglequotedstring]() { + goto l249 + } + add(rulePegText, position250) + } + if buffer[position] != rune('\'') { + goto l249 + } + position++ + { + add(ruleAction46, position) + } + goto l245 + l249: + position, tokenIndex = position245, tokenIndex245 + if buffer[position] != rune('"') { + goto l243 + } + position++ + { + position252 := position + if !_rules[ruledoublequotedstring]() { + goto l243 + } + add(rulePegText, position252) + } + if buffer[position] != rune('"') { + goto l243 + } + position++ + { + add(ruleAction47, position) + } } - l240: - add(rulecol, position239) + l245: + add(rulecol, position244) } return true - l238: - position, tokenIndex = position238, tokenIndex238 + l243: + position, tokenIndex = position243, tokenIndex243 return false }, - /* 21 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ + /* 21 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ nil, /* 22 open <- <('(' sp)> */ func() bool { - position250, tokenIndex250 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex { - position251 := position + position256 := position if buffer[position] != rune('(') { - goto l250 + goto l255 } position++ if !_rules[rulesp]() { - goto l250 + goto l255 } - add(ruleopen, position251) + add(ruleopen, position256) } return true - l250: - position, tokenIndex = position250, tokenIndex250 + l255: + position, tokenIndex = position255, tokenIndex255 return false }, /* 23 close <- <(')' sp)> */ func() bool { - position252, tokenIndex252 := position, tokenIndex + position257, tokenIndex257 := position, tokenIndex { - position253 := position + position258 := position if buffer[position] != rune(')') { - goto l252 + goto l257 } position++ if !_rules[rulesp]() { - goto l252 + goto l257 } - add(ruleclose, position253) + add(ruleclose, position258) } return true - l252: - position, tokenIndex = position252, tokenIndex252 + l257: + position, tokenIndex = position257, tokenIndex257 return false }, /* 24 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position255 := position - l256: + position260 := position + l261: { - position257, tokenIndex257 := position, tokenIndex + position262, tokenIndex262 := position, tokenIndex { - position258, tokenIndex258 := position, tokenIndex + position263, tokenIndex263 := position, tokenIndex if buffer[position] != rune(' ') { - goto l259 + goto l264 } position++ - goto l258 - l259: - position, tokenIndex = position258, tokenIndex258 + goto l263 + l264: + position, tokenIndex = position263, tokenIndex263 if buffer[position] != rune('\t') { - goto l260 + goto l265 } position++ - goto l258 - l260: - position, tokenIndex = position258, tokenIndex258 + goto l263 + l265: + position, tokenIndex = position263, tokenIndex263 if buffer[position] != rune('\n') { - goto l257 + goto l262 } position++ } - l258: - goto l256 - l257: - position, tokenIndex = position257, tokenIndex257 + l263: + goto l261 + l262: + position, tokenIndex = position262, tokenIndex262 } - add(rulesp, position255) + add(rulesp, position260) } return true }, /* 25 comma <- <(sp ',' sp)> */ func() bool { - position261, tokenIndex261 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position262 := position + position267 := position if !_rules[rulesp]() { - goto l261 + goto l266 } if buffer[position] != rune(',') { - goto l261 + goto l266 } position++ if !_rules[rulesp]() { - goto l261 + goto l266 } - add(rulecomma, position262) + add(rulecomma, position267) } return true - l261: - position, tokenIndex = position261, tokenIndex261 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 26 lbrack <- <('[' sp)> */ @@ -2603,207 +2658,207 @@ func (p *PQL) Init() { nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position265, tokenIndex265 := position, tokenIndex + position270, tokenIndex270 := position, tokenIndex { - position266 := position + position271 := position { - position267, tokenIndex267 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l268 + goto l273 } position++ - goto l267 - l268: - position, tokenIndex = position267, tokenIndex267 + goto l272 + l273: + position, tokenIndex = position272, tokenIndex272 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l265 + goto l270 } position++ } - l267: - l269: + l272: + l274: { - position270, tokenIndex270 := position, tokenIndex + position275, tokenIndex275 := position, tokenIndex { - position271, tokenIndex271 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l272 + goto l277 } position++ - goto l271 - l272: - position, tokenIndex = position271, tokenIndex271 + goto l276 + l277: + position, tokenIndex = position276, tokenIndex276 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l273 + goto l278 } position++ - goto l271 - l273: - position, tokenIndex = position271, tokenIndex271 + goto l276 + l278: + position, tokenIndex = position276, tokenIndex276 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l270 + goto l275 } position++ } - l271: - goto l269 - l270: - position, tokenIndex = position270, tokenIndex270 + l276: + goto l274 + l275: + position, tokenIndex = position275, tokenIndex275 } - add(ruleIDENT, position266) + add(ruleIDENT, position271) } return true - l265: - position, tokenIndex = position265, tokenIndex265 + l270: + position, tokenIndex = position270, tokenIndex270 return false }, /* 29 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position274, tokenIndex274 := position, tokenIndex + position279, tokenIndex279 := position, tokenIndex { - position275 := position + position280 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('-') { - goto l274 + goto l279 } position++ { - position276, tokenIndex276 := position, tokenIndex + position281, tokenIndex281 := position, tokenIndex if buffer[position] != rune('0') { - goto l277 + goto l282 } position++ - goto l276 - l277: - position, tokenIndex = position276, tokenIndex276 + goto l281 + l282: + position, tokenIndex = position281, tokenIndex281 if buffer[position] != rune('1') { - goto l274 + goto l279 } position++ } - l276: + l281: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('-') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune('T') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if buffer[position] != rune(':') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l279 } position++ - add(ruletimestampbasicfmt, position275) + add(ruletimestampbasicfmt, position280) } return true - l274: - position, tokenIndex = position274, tokenIndex274 + l279: + position, tokenIndex = position279, tokenIndex279 return false }, /* 30 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position278, tokenIndex278 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position279 := position + position284 := position { - position280, tokenIndex280 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex if buffer[position] != rune('"') { - goto l281 + goto l286 } position++ { - position282 := position + position287 := position if !_rules[ruletimestampbasicfmt]() { - goto l281 + goto l286 } - add(rulePegText, position282) + add(rulePegText, position287) } if buffer[position] != rune('"') { - goto l281 + goto l286 } position++ - goto l280 - l281: - position, tokenIndex = position280, tokenIndex280 + goto l285 + l286: + position, tokenIndex = position285, tokenIndex285 if buffer[position] != rune('\'') { - goto l283 + goto l288 } position++ { - position284 := position + position289 := position + if !_rules[ruletimestampbasicfmt]() { + goto l288 + } + add(rulePegText, position289) + } + if buffer[position] != rune('\'') { + goto l288 + } + position++ + goto l285 + l288: + position, tokenIndex = position285, tokenIndex285 + { + position290 := position if !_rules[ruletimestampbasicfmt]() { goto l283 } - add(rulePegText, position284) - } - if buffer[position] != rune('\'') { - goto l283 - } - position++ - goto l280 - l283: - position, tokenIndex = position280, tokenIndex280 - { - position285 := position - if !_rules[ruletimestampbasicfmt]() { - goto l278 - } - add(rulePegText, position285) + add(rulePegText, position290) } } - l280: - add(ruletimestampfmt, position279) + l285: + add(ruletimestampfmt, position284) } return true - l278: - position, tokenIndex = position278, tokenIndex278 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, - /* 31 timestamp <- <( Action49)> */ + /* 31 timestamp <- <( Action51)> */ nil, /* 33 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2833,78 +2888,82 @@ func (p *PQL) Init() { nil, /* 46 Action13 <- <{p.endCall()}> */ nil, + /* 47 Action14 <- <{p.startCall("Rows")}> */ nil, - /* 48 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action15 <- <{p.endCall()}> */ nil, - /* 49 Action15 <- <{ p.endCall() }> */ nil, - /* 50 Action16 <- <{ p.addBTWN() }> */ + /* 50 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 51 Action17 <- <{ p.addLTE() }> */ + /* 51 Action17 <- <{ p.endCall() }> */ nil, - /* 52 Action18 <- <{ p.addGTE() }> */ + /* 52 Action18 <- <{ p.addBTWN() }> */ nil, - /* 53 Action19 <- <{ p.addEQ() }> */ + /* 53 Action19 <- <{ p.addLTE() }> */ nil, - /* 54 Action20 <- <{ p.addNEQ() }> */ + /* 54 Action20 <- <{ p.addGTE() }> */ nil, - /* 55 Action21 <- <{ p.addLT() }> */ + /* 55 Action21 <- <{ p.addEQ() }> */ nil, - /* 56 Action22 <- <{ p.addGT() }> */ + /* 56 Action22 <- <{ p.addNEQ() }> */ nil, - /* 57 Action23 <- <{p.startConditional()}> */ + /* 57 Action23 <- <{ p.addLT() }> */ nil, - /* 58 Action24 <- <{p.endConditional()}> */ + /* 58 Action24 <- <{ p.addGT() }> */ nil, - /* 59 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action25 <- <{p.startConditional()}> */ nil, - /* 60 Action26 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action26 <- <{p.endConditional()}> */ nil, /* 61 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 62 Action28 <- <{ p.startList() }> */ + /* 62 Action28 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 63 Action29 <- <{ p.endList() }> */ + /* 63 Action29 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action30 <- <{ p.addVal(nil) }> */ + /* 64 Action30 <- <{ p.startList() }> */ nil, - /* 65 Action31 <- <{ p.addVal(true) }> */ + /* 65 Action31 <- <{ p.endList() }> */ nil, - /* 66 Action32 <- <{ p.addVal(false) }> */ + /* 66 Action32 <- <{ p.addVal(nil) }> */ nil, - /* 67 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 67 Action33 <- <{ p.addVal(true) }> */ nil, - /* 68 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 68 Action34 <- <{ p.addVal(false) }> */ nil, - /* 69 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 70 Action36 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 70 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 71 Action37 <- <{ p.addVal(p.endCall()) }> */ + /* 71 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 72 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action38 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 73 Action39 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 73 Action39 <- <{ p.addVal(p.endCall()) }> */ nil, /* 74 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 75 Action41 <- <{ p.addField(buffer[begin:end]) }> */ + /* 75 Action41 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 76 Action42 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 76 Action42 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action43 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 78 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action44 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 79 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 80 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 80 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 81 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 81 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 82 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 82 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 83 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 83 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 84 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 85 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/roaring/roaring.go b/roaring/roaring.go index a2ddecb62..abc4e4d2f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -52,16 +52,20 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - //containerArray indicates a container of bit position values + // containerArray indicates a container of bit position values containerArray = byte(1) - //containerBitmap indicates a container of bits packed in a uint64 array block + // containerBitmap indicates a container of bits packed in a uint64 array block containerBitmap = byte(2) - //containerRun indicates a container of run encoded bits + // containerRun indicates a container of run encoded bits containerRun = byte(3) maxContainerVal = 0xffff + + // maxContainerKey is the key representing the last container in a full row. + // It is the full bitmap space (2^64) divided by container width (2^16). + maxContainerKey = (1 << 48) - 1 ) type Containers interface { @@ -163,9 +167,8 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { } // Apply to the in-memory bitmap. - if op.apply(b) { + if b.DirectAdd(v) { changed = true - } } @@ -233,6 +236,18 @@ func (b *Bitmap) Count() (n uint64) { return b.Containers.Count() } +// Size returns the number of bytes required for the bitmap. +func (b *Bitmap) Size() int { + numbytes := 0 + citer, _ := b.Containers.Iterator(0) + for citer.Next() { + _, c := citer.Value() + numbytes += c.size() + + } + return numbytes +} + // CountRange returns the number of bits set between [start, end). func (b *Bitmap) CountRange(start, end uint64) (n uint64) { if b.Containers.Size() == 0 { @@ -742,6 +757,38 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { return output } +// Shift shifts the contents of b by 1. +func (b *Bitmap) Shift(n int) (*Bitmap, error) { + if n != 1 { + return nil, errors.New("cannot shift by a value other than 1") + } + output := NewBitmap() + iiter, _ := b.Containers.Iterator(0) + lastCarry := false + lastKey := uint64(0) + for iiter.Next() { + ki, ci := iiter.Value() + o, carry := shift(ci) + if lastCarry { + o.add(0) + } + if o.n > 0 { + output.Containers.Put(ki, o) + } + lastCarry = carry + lastKey = ki + } + // As long as the carry wasn't from the max container, + // append a new container and add the carried bit. + if lastCarry && lastKey != maxContainerKey { + extra := NewContainer() + extra.add(0) + output.Containers.Put(lastKey+1, extra) + } + + return output, nil +} + // removeEmptyContainers deletes all containers that have a count of zero. func (b *Bitmap) removeEmptyContainers() { citer, _ := b.Containers.Iterator(0) @@ -3350,6 +3397,88 @@ func xorBitmapBitmap(a, b *Container) *Container { return output } +// shift() shifts the contents of c by one. It returns +// the new container and a bool indicating whether a +// carry bit was shifted out. +func shift(c *Container) (*Container, bool) { + if c.isArray() { + return shiftArray(c) + } else if c.isRun() { + return shiftRun(c) + } + return shiftBitmap(c) +} + +// shiftArray is an array-specific implementation of shift(). +func shiftArray(a *Container) (*Container, bool) { + statsHit("shift/Array") + carry := false + output := &Container{containerType: containerArray} + output.array = make([]uint16, len(a.array)) + output.array = output.array[:0] + output.n = a.n + for _, v := range a.array { + if v+1 == 0 { // overflow + carry = true + output.n -= 1 + } else { + output.array = append(output.array, v+1) + } + } + return output, carry +} + +// shiftBitmap is a bitmap-specific implementation of shift(). +func shiftBitmap(a *Container) (*Container, bool) { + statsHit("shift/Bitmap") + carry := false + output := &Container{containerType: containerBitmap} + output.bitmap = make([]uint64, len(a.bitmap)) + output.bitmap = output.bitmap[:0] + output.n = a.n + lastCarry := false + for _, v := range a.bitmap { + carry = (v & (1 << 63)) != 0 + v = v << 1 + if lastCarry { + v |= 1 + } + output.bitmap = append(output.bitmap, v) + lastCarry = carry + } + if carry { + output.n -= 1 + } + return output, carry +} + +// shiftRun is a run-specific implementation of shift(). +func shiftRun(a *Container) (*Container, bool) { + statsHit("shift/Run") + carry := false + output := &Container{containerType: containerRun} + output.runs = make([]interval16, len(a.runs)) + output.runs = output.runs[:0] + for _, v := range a.runs { + if v.start+1 == 0 { // final run was 1 bit on container edge + carry = true + output.n -= 1 + break + } else if v.last+1 == 0 { // final run ends on container edge + v.start += 1 + carry = true + output.n -= 1 + } else { + v.start += 1 + v.last += 1 + carry = false + } + output.runs = append(output.runs, v) + } + + return output, carry +} + // opType represents a type of operation. type opType uint8 @@ -3869,7 +3998,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint header = pos if size > (1 << 16) { - err = fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + err = fmt.Errorf("it is logically impossible to have more than (1<<16) containers") return size, containerTyper, header, pos, haveRuns, err } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index cd22978fb..f99d922e8 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -106,6 +106,24 @@ func bitmapFirstBitSet() []uint64 { return bitmap } +func bitmapSecondBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000002 + return bitmap +} + +func bitmapLastBitFirstRowSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitSecoundRowSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[1] = 0x0000000000000001 + return bitmap +} + func bitmapLastBitSet() []uint64 { bitmap := make([]uint64, bitmapN) bitmap[bitmapN-1] = 0x8000000000000000 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a5157a97d..5ca0edb2b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3224,11 +3224,11 @@ func TestContainerCombinations(t *testing.T) { //func getFunc(func(a, b *container) *container, m, n *container) *container { func runContainerFunc(f interface{}, c ...*Container) *Container { - switch f.(type) { + switch f := f.(type) { case func(*Container) *Container: - return f.(func(*Container) *Container)(c[0]) + return f(c[0]) case func(*Container, *Container) *Container: - return f.(func(a, b *Container) *Container)(c[0], c[1]) + return f(c[0], c[1]) } return nil } @@ -3317,3 +3317,123 @@ func TestEquals(t *testing.T) { } } */ +func TestShiftArray(t *testing.T) { + a := &Container{ + containerType: containerArray, + } + tests := []struct { + array []uint16 + exp []uint16 + }{ + { + array: []uint16{1}, + exp: []uint16{2}, + }, + { + array: []uint16{}, + exp: []uint16{}, + }, + { + array: []uint16{1, 2, 3, 4, 5, 11, 12}, + exp: []uint16{2, 3, 4, 5, 6, 12, 13}, + }, + { + array: []uint16{65535}, + exp: []uint16{}, + }, + } + + for i, test := range tests { + a.array = test.array + a.n = int32(len(a.array)) + ret1, _ := shift(a) // test generic shift function + ret2, _ := shiftArray(a) // test array-specific shift function + if !reflect.DeepEqual(ret1.array, test.exp) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array) + } else if !reflect.DeepEqual(ret2.array, test.exp) { + t.Fatalf("test #%v shiftArray() expected %v, but got %v", i, test.exp, ret2.array) + } + } +} + +func TestShiftBitmap(t *testing.T) { + a := &Container{ + containerType: containerBitmap, + } + tests := []struct { + bitmap []uint64 + exp []uint64 + }{ + { + bitmap: bitmapFirstBitSet(), + exp: bitmapSecondBitSet(), + }, + { + bitmap: bitmapLastBitSet(), + exp: bitmapEmpty(), + }, + { + bitmap: bitmapLastBitFirstRowSet(), + exp: bitmapFirstBitSecoundRowSet(), + }, + } + + for i, test := range tests { + a.bitmap = test.bitmap + a.n = 1 + ret1, _ := shift(a) // test generic shift function + ret2, _ := shiftBitmap(a) // test bitmap-specific shift function + if !reflect.DeepEqual(ret1.bitmap, test.exp) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.bitmap) + } else if !reflect.DeepEqual(ret2.bitmap, test.exp) { + t.Fatalf("test #%v shiftBitmap() expected %v, but got %v", i, test.exp, ret2.bitmap) + } + } +} +func TestShiftRun(t *testing.T) { + a := &Container{ + containerType: containerRun, + } + + tests := []struct { + runs []interval16 + n int32 + en int32 + exp []interval16 + carry bool + }{ + { + runs: []interval16{{start: 5, last: 10}}, + n: 5, + en: 5, + exp: []interval16{{start: 6, last: 11}}, + carry: false, + }, + { + runs: []interval16{{start: 5, last: 65535}}, + n: 65530, + en: 65529, + exp: []interval16{{start: 6, last: 65535}}, + carry: true, + }, + { + runs: []interval16{{start: 65535, last: 65535}}, + n: 1, + en: 0, + exp: []interval16{}, + carry: true, + }, + } + + for i, test := range tests { + a.runs = test.runs + a.n = test.n + ret1, c1 := shift(a) // test generic shift function + ret2, c2 := shiftRun(a) // test run-specific shift function + if !reflect.DeepEqual(ret1.runs, test.exp) && c1 == test.carry && ret1.n == test.en { + t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs, ret1.n) + } else if !reflect.DeepEqual(ret2.runs, test.exp) && c2 == test.carry && ret2.n == test.en { + t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs, ret2.n) + } + } +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 4c3d15a4d..a4333d0fc 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -37,6 +37,29 @@ func TestContainerCount(t *testing.T) { t.Fatalf("Count != CountRange\n") } } +func TestSize(t *testing.T) { + //array + a := roaring.NewFileBitmap(0, 65535, 131072) + if a.Size() != 6 { + t.Fatalf("Size in bytes incorrect \n") + } + + //bitmap + b := roaring.NewFileBitmap() + for i := uint64(0); i <= 4096; i++ { + b.DirectAdd(i) + } + + if b.Size() != 8192 { + t.Fatalf("Size in bytes incorrect \n") + } + //convert to rle + b.Optimize() + //rle + if b.Size() != 6 { + t.Fatalf("Size in bytes incorrect \n") + } +} func TestCountRange(t *testing.T) { tests := []struct { @@ -962,6 +985,18 @@ func TestBitmap_IntersectionCount_Mixed(t *testing.T) { } } +func TestBitmap_Shift(t *testing.T) { + var max uint64 = math.MaxUint64 + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536, max) + bm2 := roaring.NewFileBitmap(1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 65537) + + if got, err := bm1.Shift(1); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got.Slice(), bm2.Slice()) { + t.Fatalf("unexpected bitmap: expected %v, but got %v", bm2.Slice(), got.Slice()) + } +} + func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) } @@ -1498,6 +1533,41 @@ func BenchmarkSliceDescending(b *testing.B) { for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { bm.Add(col) } + bm.Add(0) + } +} + +func BenchmarkSliceAscendingStriped(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewFileBitmap() + l := uint64(pilosa.ShardWidth / 8) + for col := uint64(0); col < l; col++ { + bm.Add(l*0 + col) + bm.Add(l*1 + col) + bm.Add(l*2 + col) + bm.Add(l*3 + col) + bm.Add(l*4 + col) + bm.Add(l*5 + col) + bm.Add(l*6 + col) + bm.Add(l*7 + col) + } + } +} + +func BenchmarkSliceDescendingStriped(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewFileBitmap() + l := uint64(pilosa.ShardWidth / 8) + for col := uint64(l); col < l+1; col-- { + bm.Add(l*7 + col) + bm.Add(l*6 + col) + bm.Add(l*5 + col) + bm.Add(l*4 + col) + bm.Add(l*3 + col) + bm.Add(l*2 + col) + bm.Add(l*1 + col) + bm.Add(l*0 + col) + } } } diff --git a/row.go b/row.go index 255c3e847..0a8fc0acd 100644 --- a/row.go +++ b/row.go @@ -19,6 +19,7 @@ import ( "sort" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // Row is a set of integers (the associated columns), and attributes which are @@ -167,6 +168,32 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } +// Shift returns the bitwise shift of r by n bits. +// Currently only positive shift values are supported. +func (r *Row) Shift(n int64) (*Row, error) { + if n < 0 { + return nil, errors.New("cannot shift by negative values") + } else if n == 0 { + return r, nil + } + + work := r + var segments []rowSegment + for i := int64(0); i < n; i++ { + segments = segments[:0] + for _, segment := range work.segments { + shifted, err := segment.Shift() + if err != nil { + return nil, errors.Wrap(err, "shifting row segment") + } + segments = append(segments, *shifted) + } + work = &Row{segments: segments} + } + + return work, nil +} + // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) @@ -341,6 +368,21 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { } } +// Shift returns s shifted by 1 bit. +func (s *rowSegment) Shift() (*rowSegment, error) { + //TODO deal with overflow + data, err := s.data.Shift(1) + if err != nil { + return nil, errors.Wrap(err, "shifting roaring data") + } + + return &rowSegment{ + data: *data, + shard: s.shard, + n: data.Count(), + }, nil +} + // SetBit sets the i-th column of the row. func (s *rowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() diff --git a/server.go b/server.go index ca5bfcce4..4902c87d8 100644 --- a/server.go +++ b/server.go @@ -487,7 +487,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateShardMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field) + return fmt.Errorf("local field not found: %s/%s", obj.Index, obj.Field) } if err := f.AddRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { return errors.Wrap(err, "adding remote available shards") @@ -505,7 +505,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateFieldMessage: idx := s.holder.Index(obj.Index) if idx == nil { - return fmt.Errorf("Local Index not found: %s", obj.Index) + return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta _, err := idx.createField(obj.Field, *opt) @@ -525,7 +525,7 @@ func (s *Server) receiveMessage(m Message) error { case *CreateViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Field not found: %s", obj.Field) + return fmt.Errorf("local field not found: %s", obj.Field) } _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { @@ -534,7 +534,7 @@ func (s *Server) receiveMessage(m Message) error { case *DeleteViewMessage: f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Field not found: %s", obj.Field) + return fmt.Errorf("local field not found: %s", obj.Field) } err := f.deleteView(obj.View) if err != nil { diff --git a/server/cluster_test.go b/server/cluster_test.go index 90467f4b2..617bf6cad 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -399,7 +399,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { nodeID := mustNodeID(m0.URL()) resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator." + expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator" if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { diff --git a/server/config.go b/server/config.go index 6c159721e..6223f07da 100644 --- a/server/config.go +++ b/server/config.go @@ -15,10 +15,17 @@ package server import ( + "context" + "fmt" + "log" + "net" + "strconv" + "strings" "time" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/toml" + "github.com/pkg/errors" "github.com/uber/jaeger-client-go" ) @@ -37,9 +44,15 @@ type Config struct { // DataDir is the directory where Pilosa stores both indexed data and // running state such as cluster topology information. DataDir string `toml:"data-dir"` + // Bind is the host:port on which Pilosa will listen. Bind string `toml:"bind"` + // Advertise is the address advertised by the server to other nodes + // in the cluster. It should be reachable by all other nodes and should + // route to an interface that Bind is listening on. + Advertise string `toml:"advertise"` + // MaxWritesPerRequest limits the number of mutating commands that can be in // a single request to the server. This includes Set, Clear, // SetRowAttrs & SetColumnAttrs. @@ -110,22 +123,17 @@ func NewConfig() *Config { DataDir: "~/.pilosa", Bind: ":10101", MaxWritesPerRequest: 5000, - // LogPath: "", - // Verbose: false, - TLS: TLSConfig{}, + TLS: TLSConfig{}, } // Cluster config. c.Cluster.Disabled = false - // c.Cluster.Coordinator = false c.Cluster.ReplicaN = 1 c.Cluster.Hosts = []string{} c.Cluster.LongQueryTime = toml.Duration(time.Minute) // Gossip config. c.Gossip.Port = "14000" - // c.Gossip.Seeds = []string{} - // c.Gossip.Key = "" c.Gossip.StreamTimeout = toml.Duration(10 * time.Second) c.Gossip.SuspicionMult = 4 c.Gossip.PushPullInterval = toml.Duration(30 * time.Second) @@ -140,7 +148,6 @@ func NewConfig() *Config { // Metric config. c.Metric.Service = "none" - // c.Metric.Host = "" c.Metric.PollInterval = toml.Duration(0 * time.Minute) c.Metric.Diagnostics = true @@ -150,3 +157,197 @@ func NewConfig() *Config { return c } + +// validateAddrs controls the address fields in the Config object +// and fills in any blanks. +// The addresses fields must be guaranteed by the caller to either be +// completely empty, or have both a host part and a port part +// separated by a colon. In the latter case either can be empty to +// indicate it's left unspecified. +func (cfg *Config) validateAddrs(ctx context.Context) error { + // Validate the advertise address. + advScheme, advHost, advPort, err := validateAdvertiseAddr(ctx, cfg.Advertise, cfg.Bind) + if err != nil { + return errors.Wrapf(err, "validating advertise address") + } + cfg.Advertise = schemeHostPortString(advScheme, advHost, advPort) + + // Validate the listen address. + listenScheme, listenHost, listenPort, err := validateListenAddr(ctx, cfg.Bind) + if err != nil { + return errors.Wrap(err, "validating listen address") + } + cfg.Bind = schemeHostPortString(listenScheme, listenHost, listenPort) + + return nil +} + +// validateAdvertiseAddr validates and normalizes an address accessible +// Ensures that if the "host" part is empty, it gets filled in with +// the configured listen address if any, otherwise it makes a best +// guess at the outbound IP address. +// Returns scheme, host, port as strings. +func validateAdvertiseAddr(ctx context.Context, advAddr, listenAddr string) (string, string, string, error) { + listenScheme, listenHost, listenPort, err := splitAddr(listenAddr) + if err != nil { + return "", "", "", errors.Wrap(err, "getting listen address") + } + + advScheme, advHostPort := splitScheme(advAddr) + advHost, advPort := "", "" + if advHostPort != "" { + var err error + advHost, advPort, err = net.SplitHostPort(advHostPort) + if err != nil { + return "", "", "", errors.Wrapf(err, "splitting host port: %s", advHostPort) + } + } + // If no advertise scheme was specified, use the one from + // the listen address. + if advScheme == "" { + advScheme = listenScheme + } + // If there was no port number, reuse the one from the listen + // address. + if advPort == "" || advPort == "0" { + advPort = listenPort + } + // Resolve non-numeric to numeric. + portNumber, err := net.DefaultResolver.LookupPort(ctx, "tcp", advPort) + if err != nil { + return "", "", "", errors.Wrapf(err, "looking up non-numeric port: %v", advPort) + } + advPort = strconv.Itoa(portNumber) + + // If the advertise host is empty, then we have two cases. + if advHost == "" { + if listenHost == "0.0.0.0" { + advHost = outboundIP().String() + } else { + advHost = listenHost + } + } + return advScheme, advHost, advPort, nil +} + +// outboundIP gets the preferred outbound ip of this machine. +func outboundIP() net.IP { + // This is not actually making a connection to 8.8.8.8. + // net.Dial() selects the IP address that would be used + // if an actual connection to 8.8.8.8 were made, so this + // choice of address is just meant to ensure that an + // external address is returned (as opposed to a local + // address like 127.0.0.1). + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*net.UDPAddr) + + return localAddr.IP +} + +// validateListenAddr validates and normalizes an address suitable for +// use with net.Listen(). This accepts an empty "host" part to signify +// the default (localhost) should be used. Rresolves host names to IP +// addresses. +// Returns scheme, host, port as strings. +func validateListenAddr(ctx context.Context, addr string) (string, string, string, error) { + scheme, host, port, err := splitAddr(addr) + if err != nil { + return "", "", "", errors.Wrap(err, "getting listen address") + } + rHost, rPort, err := resolveAddr(ctx, host, port) + if err != nil { + return "", "", "", errors.Wrap(err, "resolving address") + } + return scheme, rHost, rPort, nil +} + +// splitScheme returns two strings: the scheme and the hostPort. +func splitScheme(addr string) (string, string) { + parts := strings.SplitN(addr, "://", 2) + if len(parts) == 1 { + return "", addr + } + return parts[0], parts[1] +} + +func schemeHostPortString(scheme, host, port string) string { + var s string + if scheme != "" { + s += fmt.Sprintf("%s://", scheme) + } + return s + net.JoinHostPort(host, port) +} + +// splitAddr returns scheme, host, port as strings. +func splitAddr(addr string) (string, string, string, error) { + scheme, hostPort := splitScheme(addr) + host, port := "", "" + if hostPort != "" { + var err error + host, port, err = net.SplitHostPort(hostPort) + if err != nil { + return "", "", "", errors.Wrapf(err, "splitting host port: %s", hostPort) + } + } + // It's not ideal to have a default here, but the alterative + // results in a port of 0, which causes Pilosa to listen on + // a random port. + if port == "" { + port = "10101" + } + return scheme, host, port, nil +} + +// resolveAddr resolves non-numeric addresses to numeric (IP, port) addresses. +func resolveAddr(ctx context.Context, host, port string) (string, string, error) { + resolver := net.DefaultResolver + + // Resolve the port number. This may translate service names + // e.g. "postgresql" to a numeric value. + portNumber, err := resolver.LookupPort(ctx, "tcp", port) + if err != nil { + return "", "", errors.Wrapf(err, "resolving up port: %v", port) + } + port = strconv.Itoa(portNumber) + + // Resolve the address. + if host == "" || host == "localhost" { + return host, port, nil + } + + addr, err := lookupAddr(ctx, resolver, host) + if err != nil { + return "", "", errors.Wrap(err, "looking up address") + } + return addr, port, nil +} + +// lookupAddr resolves the given address/host to an IP address. If +// multiple addresses are resolved, it returns the first IPv4 address +// available if there is one, otherwise the first address. +func lookupAddr(ctx context.Context, resolver *net.Resolver, host string) (string, error) { + // Resolve the IP address or hostname to an IP address. + addrs, err := resolver.LookupIPAddr(ctx, host) + if err != nil { + return "", errors.Wrap(err, "looking up IP addresses") + } + if len(addrs) == 0 { + return "", fmt.Errorf("cannot resolve %q to an address", host) + } + + // LookupIPAddr() can return a mix of IPv6 and IPv4 + // addresses. Return the first IPv4 address if possible. + for _, addr := range addrs { + if ip := addr.IP.To4(); ip != nil { + return ip.String(), nil + } + } + + // No IPv4 address, return the first resolved address instead. + return addrs[0].String(), nil +} diff --git a/server/config_internal_test.go b/server/config_internal_test.go new file mode 100644 index 000000000..2b29ea51e --- /dev/null +++ b/server/config_internal_test.go @@ -0,0 +1,153 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "net" + "os" + "strings" + "testing" +) + +type addrs struct{ bind, advertise string } + +func TestConfig_validateAddrs(t *testing.T) { + + // Prepare some reference strings that will be checked in the + // test below. + outboundAddr := outboundIP().String() + hostname, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + hostAddr, err := lookupAddr(context.Background(), net.DefaultResolver, hostname) + if err != nil { + t.Fatal(err) + } + if strings.Contains(hostAddr, ":") { + hostAddr = "[" + hostAddr + "]" + } + + tests := []struct { + expErr string + in addrs + exp addrs + }{ + // Default values; addresses set empty. + {"", + addrs{"", ""}, + addrs{":10101", ":10101"}}, + {"", + addrs{":", ""}, + addrs{":10101", ":10101"}}, + {"", + addrs{"", ":"}, + addrs{":10101", ":10101"}}, + {"", + addrs{":", ":"}, + addrs{":10101", ":10101"}}, + // Listener :port. + {"", + addrs{":1234", ""}, + addrs{":1234", ":1234"}}, + // Listener with host:port. + {"", + addrs{hostAddr + ":10101", ""}, + addrs{hostAddr + ":10101", hostAddr + ":10101"}}, + // Listener with host:. + {"", + addrs{hostAddr + ":", ""}, + addrs{hostAddr + ":10101", hostAddr + ":10101"}}, + // Listener with scheme:. + {"", + addrs{"http://" + hostAddr + ":", ""}, + addrs{"http://" + hostAddr + ":10101", "http://" + hostAddr + ":10101"}}, + // Listener with localhost:port. + {"", + addrs{"localhost:1234", ""}, + addrs{"localhost:1234", "localhost:1234"}}, + // Listener with localhost:. + {"", + addrs{"localhost:", ""}, + addrs{"localhost:10101", "localhost:10101"}}, + // Listener and advertise addresses. + {"", + addrs{hostAddr + ":1234", hostAddr + ":"}, + addrs{hostAddr + ":1234", hostAddr + ":1234"}}, + // Explicit port number in advertise addr. + {"", + addrs{hostAddr + ":1234", hostAddr + ":7890"}, + addrs{hostAddr + ":1234", hostAddr + ":7890"}}, + // Use a non-numeric port number. + {"", + addrs{":postgresql", ""}, + addrs{":5432", ":5432"}}, + // Advertise port 0 means reuse listen port. + {"", + addrs{":1234", ":0"}, + addrs{":1234", ":1234"}}, + // Listen on all interfaces. Determine advertise address. + {"", + addrs{"0.0.0.0:1234", ""}, + addrs{"0.0.0.0:1234", outboundAddr + ":1234"}}, + // Expected errors. + + // Missing port number. + {"missing port in address", + addrs{"localhost", ""}, + addrs{}}, + {"missing port in address", + addrs{":1234", "localhost"}, + addrs{}}, + // Invalid port number. + {"invalid port", + addrs{"localhost:-1234", ""}, + addrs{}}, + {"validating advertise address", + addrs{"localhost:foo", ""}, + addrs{}}, + {"no such host", + addrs{"333.333.333.333:1234", ""}, + addrs{}}, + } + + for i, test := range tests { + c := NewConfig() + + c.Bind = test.in.bind + c.Advertise = test.in.advertise + + err := c.validateAddrs(context.Background()) + + if err != nil && test.expErr == "" { + t.Fatal(err) + } else if err == nil && test.expErr != "" { + t.Fatalf("test %d: expected error string to contain %s, but got no error", i, test.expErr) + } else if err != nil && test.expErr != "" { + if strings.Contains(err.Error(), test.expErr) { + continue + } else { + t.Fatalf("test %d: expected error string to contain %s, but got %s", i, test.expErr, err.Error()) + } + } + + if c.Bind != test.exp.bind { + t.Fatalf("test %d: bind address: expected %s, but got %s", i, test.exp.bind, c.Bind) + } else if c.Advertise != test.exp.advertise { + t.Fatalf("test %d: advertise address: expected %s, but got %s", i, test.exp.advertise, c.Advertise) + } + } +} diff --git a/server/server.go b/server/server.go index c80f8268e..836b7fec7 100644 --- a/server/server.go +++ b/server/server.go @@ -21,6 +21,7 @@ package server import ( "bytes" + "context" "crypto/tls" "io" "log" @@ -78,6 +79,7 @@ type Command struct { Handler pilosa.Handler API *pilosa.API ln net.Listener + listenURI *pilosa.URI closeTimeout time.Duration serverOptions []pilosa.ServerOption @@ -151,7 +153,7 @@ func (m *Command) Start() (err error) { return errors.Wrap(err, "opening server") } - m.logger.Printf("listening as %s\n", m.API.Node().URI) + m.logger.Printf("listening as %s\n", m.listenURI) return nil } @@ -187,6 +189,15 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) + // validateAddrs sets the appropriate values for Bind and Advertise + // based on the inputs. It is not responsible for applying defaults, although + // it does provide a non-zero port (10101) in the case where no port is specified. + // The alternative would be to use port 0, which would choose a random port, but + // currently that's not what we want. + if err := m.Config.validateAddrs(context.Background()); err != nil { + return errors.Wrap(err, "validating addresses") + } + uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { return errors.Wrap(err, "processing bind address") @@ -231,8 +242,20 @@ func (m *Command) SetupServer() error { uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) } + // Save listenURI for later reference. + m.listenURI = uri + c := http.GetHTTPClient(TLSConfig) + // Get advertise address as uri. + advertiseURI, err := pilosa.AddressWithDefaults(m.Config.Advertise) + if err != nil { + return errors.Wrap(err, "processing advertise address") + } + if advertiseURI.Port == 0 { + advertiseURI.SetPort(uri.Port) + } + // Primary store configuration is handled automatically now. if m.Config.Translation.PrimaryURL != "" { m.logger.Printf("DEPRECATED: The primary-url configuration option is no longer used.") @@ -258,7 +281,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), pilosa.OptServerStatsClient(statsClient), - pilosa.OptServerURI(uri), + pilosa.OptServerURI(advertiseURI), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStoreFunc(http.NewTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), @@ -295,7 +318,6 @@ func (m *Command) SetupServer() error { http.OptHandlerCloseTimeout(m.closeTimeout), ) return errors.Wrap(err, "new handler") - } // setupNetworking sets up internode communication based on the configuration. @@ -310,7 +332,7 @@ func (m *Command) setupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.API.Node().URI.Host + gossipHost := m.listenURI.Host m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return errors.Wrap(err, "getting transport") @@ -320,6 +342,7 @@ func (m *Command) setupNetworking() error { m.Config.Gossip, m.API, gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}), + gossip.WithPilosaLogger(m.logger), gossip.WithTransport(m.gossipTransport), ) if err != nil { diff --git a/server/server_test.go b/server/server_test.go index a2fb5b5d7..3b6be7efa 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -293,7 +293,7 @@ func TestMain_GroupBy(t *testing.T) { } // Query row. - if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil { + if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil { t.Fatal(err) } else { test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) @@ -468,7 +468,6 @@ func TestClusteringNodesReplica1(t *testing.T) { // Create new main with the same config. config := cluster[2].Command.Config config.Translation.MapSize = 100000 - // config.Bind = cluster[2].API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port)) diff --git a/server_internal_test.go b/server_internal_test.go index e22681764..a8af8186c 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -38,7 +38,7 @@ func TestCountOpenFiles(t *testing.T) { func TestMonitorAntiEntropyZero(t *testing.T) { - td, err := ioutil.TempDir("", "") + td, err := ioutil.TempDir(*TempDir, "") if err != nil { t.Fatalf("getting temp dir: %v", err) } diff --git a/translate.go b/translate.go index b86041e3d..db129cd8f 100644 --- a/translate.go +++ b/translate.go @@ -372,7 +372,6 @@ func (s *TranslateFile) monitorReplication() { if err := s.replicate(ctx); err != nil { s.logger.Printf("pilosa: replication error: %s", err) } - select { case <-ctx.Done(): return @@ -412,22 +411,42 @@ func (s *TranslateFile) replicate(ctx context.Context) error { // Wrap in bufferred I/O so it implements io.ByteReader. bufr := bufio.NewReader(rc) + // we need a way to make an asynchronous routine hand us back an error, + // but we might not still be there to get it. so we have a buffer. + chErr := make(chan error, 1) + // Continually read new entries from primary and append to local store. for { // Read next available entry. var entry LogEntry - if _, err := entry.ReadFrom(bufr); err == io.EOF { + if _, err = entry.ReadFrom(bufr); err == io.EOF { return nil } else if err != nil { return err } - s.mu.Lock() - // Write to local store. - if err := s.appendEntry(&entry); err != nil { - s.mu.Unlock() - return err + // note: we should never end up spawning two of this goroutine + // at once. either we end up reading the error from chErr below, + // and this loop continues, or we don't, and the whole function + // returns. if the function returns, we can write that single + // error to the empty channel with a buffer of 1, the goroutine + // terminates, and chErr becomes garbage-collectable. + go func() { + s.mu.Lock() + defer s.mu.Unlock() + // Write to local store. + err = s.appendEntry(&entry) + chErr <- err + }() + select { + case err = <-chErr: + if err != nil { + return err + } + case <-s.replicationClosing: + return nil + case <-ctx.Done(): + return nil } - s.mu.Unlock() } } diff --git a/translate_test.go b/translate_test.go index fe78108ff..2b5d35a49 100644 --- a/translate_test.go +++ b/translate_test.go @@ -803,7 +803,7 @@ type TranslateFile struct { } func NewTranslateFile() *TranslateFile { - f, err := ioutil.TempFile("", "") + f, err := ioutil.TempFile(*TempDir, "") if err != nil { panic(err) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 21d8d0780..016d5276f 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -212,7 +212,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) t.common.Nodes = append(t.common.Nodes, node) // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i)) if err != nil { return nil, err } diff --git a/view_internal_test.go b/view_internal_test.go index 4ca7667ad..6867b03ed 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -24,7 +24,7 @@ import ( // mustOpenView returns a new instance of View with a temporary path. func mustOpenView(index, field, name string) *view { - path, err := ioutil.TempDir("", "pilosa-view-") + path, err := ioutil.TempDir(*TempDir, "pilosa-view-") if err != nil { panic(err) }