diff --git a/.circleci/config.yml b/.circleci/config.yml index c5bf059e2..bd1ca86cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,9 +68,20 @@ jobs: docker: - image: circleci/python:2.7-jessie steps: + - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout - run: sudo pip install awscli - run: make prerelease-upload + dockerhub-upload: + <<: *defaults + steps: + - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR + - *fast-checkout + - setup_remote_docker + - run: make docker + - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push pilosa/pilosa:master workflows: version: 2 test: @@ -104,3 +115,7 @@ workflows: - prerelease-upload: requires: - prerelease + - dockerhub-upload: + requires: + - linter + - test-golang-1.10 diff --git a/api.go b/api.go index 7736b12e5..7c475b473 100644 --- a/api.go +++ b/api.go @@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return QueryResponse{}, errors.Wrap(err, "validating api method") } - resp := QueryResponse{} - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - return resp, errors.Wrap(err, "parsing") + return QueryResponse{}, errors.Wrap(err, "parsing") } execOpts := &execOptions{ Remote: req.Remote, @@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. } - results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) + resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { - return resp, errors.Wrap(err, "executing") + return QueryResponse{}, errors.Wrap(err, "executing") } - resp.Results = results - // Fill column attributes if requested. - // execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true) - if execOpts.ColumnAttrs { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Row) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs) - if err != nil { - return resp, errors.Wrap(err, "reading column attrs") - } - - // Translate column attributes, if necessary. - if api.holder.translateFile != nil { - for _, col := range resp.ColumnAttrSets { - v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID) - if err != nil { - return resp, err - } - col.Key, col.ID = v, 0 - } - } - - resp.ColumnAttrSets = columnAttrSets - } return resp, nil } -// readColumnAttrSets returns a list of column attribute objects by id. -func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { - if index == nil { - return nil, nil - } - - ax := make([]*ColumnAttrSet, 0, len(ids)) - for _, id := range ids { - // Read attributes for column. Skip column if empty. - attrs, err := index.ColumnAttrStore().Attrs(id) - if err != nil { - return nil, errors.Wrap(err, "getting attrs") - } else if len(attrs) == 0 { - continue - } - - // Append column with attributes. - ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) - } - - return ax, nil -} - // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { @@ -321,13 +263,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, nodes := api.cluster.shardNodes(indexName, shard) var eg errgroup.Group + field := api.holder.Field(indexName, fieldName) + if field == nil { + return newNotFoundError(ErrFieldNotFound) + } + + // only set fields are supported + if field.Type() != FieldTypeSet { + return NewBadRequestError(errors.New("roaring import is only supported for set fields")) + } + for _, node := range nodes { node := node if node.ID == api.server.nodeID { - field := api.holder.Field(indexName, fieldName) - if field == nil { - return newNotFoundError(ErrFieldNotFound) - } // must make a copy of data to operate on locally. field.importRoaring changes data d2 := make([]byte, len(data)) copy(d2, data) @@ -732,7 +680,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error { if ts == 0 { continue } - t := time.Unix(0, ts) + t := time.Unix(0, ts).UTC() timestamps[i] = &t } @@ -801,6 +749,8 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error { } // MaxShards returns the maximum shard number for each index in a map. +// TODO (2.0): This method has been deprecated. Instead, use +// AvailableShardsByIndex. func (api *API) MaxShards(_ context.Context) map[string]uint64 { m := make(map[string]uint64) for k, v := range api.holder.availableShardsByIndex() { diff --git a/cache.go b/cache.go index 8ae8caefc..df82a5802 100644 --- a/cache.go +++ b/cache.go @@ -167,8 +167,10 @@ func NewRankCache(maxEntries uint32) *rankCache { func (c *rankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the column count is below the threshold. - if n < c.thresholdValue { + // Ignore if the column count is below the threshold, + // unless the count is 0, which is effectively used + // to clear the cache value. + if n < c.thresholdValue && n > 0 { return } diff --git a/cmd/server_test.go b/cmd/server_test.go index e58f8af4d..84b2297d4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) { tests := []commandTest{ // TEST 0 { - args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, + args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"}, env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" @@ -65,13 +65,14 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) + v.Check(cmd.Server.Config.Translation.MapSize, 100000) return v.Error() }, }, // TEST 1 { args: []string{"server", "--anti-entropy.interval", "9m0s"}, - env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110"}, + env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"}, cfgFileContent: ` bind = "localhost:0" data-dir = "` + actualDataDir + `" @@ -85,12 +86,13 @@ func TestServerConfig(t *testing.T) { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) + v.Check(cmd.Server.Config.Translation.MapSize, 100000) return v.Error() }, }, // TEST 2 { - args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, + args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"}, env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" diff --git a/ctl/import_test.go b/ctl/import_test.go index cd75a5d0a..2913005fb 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { t.Fatalf("Import Run with values doesn't work: %s", err) } } + +// Ensure that import into bool field runs. +func TestImportCommand_RunBool(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + ctx := context.Background() + + cmd := test.MustRunCluster(t, 1)[0] + cm.Host = cmd.API.Node().URI.HostPort() + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + + cm.Index = "i" + cm.Field = "f" + + t.Run("Valid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run to bool field doesn't work: %s", err) + } + }) + + // Ensure that invalid bool values return an error. + t.Run("Invalid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-invalid-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3\n2,4")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") { + t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err) + } + }) +} diff --git a/ctl/server.go b/ctl/server.go index 90fdfb46e..7f384ec7b 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") + flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.") // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") diff --git a/diagnostics.go b/diagnostics.go index cad7725d7..673fb0c7f 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool { return false } +// EnrichWithCPUInfo adds CPU information to the diagnostics payload. +func (d *diagnosticsCollector) EnrichWithCPUInfo() { + d.Set("CPUArch", d.server.systemInfo.CPUArch()) +} + // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *diagnosticsCollector) EnrichWithOSInfo() { uptime, err := d.server.systemInfo.Uptime() @@ -265,6 +270,7 @@ type SystemInfo interface { MemFree() (uint64, error) MemTotal() (uint64, error) MemUsed() (uint64, error) + CPUArch() string } // newNopSystemInfo creates a no-op implementation of SystemInfo. @@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) { func (n *nopSystemInfo) MemUsed() (uint64, error) { return 0, nil } + +// CPUArch returns the CPU architecture, such as amd64 +func (n *nopSystemInfo) CPUArch() string { + return "" +} diff --git a/docs/administration.md b/docs/administration.md index cd4e30184..e626cc0e5 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the pilosa import -i project -f stargazer-counts project-stargazer-counts.csv ``` +##### Importing Boolean Values + +If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true). + +For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row. +``` +0,3 +0,9 +1,1 +1,2 +1,4 +1,8 +``` +

Note that you must first create a field. View Create Field for more details. The `-e` flag can create the necessary schema when using a field of type "set".

diff --git a/docs/api-reference.md b/docs/api-reference.md index 890b37941..216a01bca 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option * `int` * `min` (int): Minimum integer value allowed for the field. * `max` (int): Maximum integer value allowed for the field. +* `bool` + * (boolean fields take no arguments) * `time` * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field. * `mutex` diff --git a/docs/configuration.md b/docs/configuration.md index 22a67ddcf..5750441ef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -307,6 +307,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h skip-verify = true ``` +#### Translation Map Size + +* Description: Size in bytes of mmap to allocate for key translation +* Flag: `translation.map-size` +* Env: `PILOSA_TRANSLATION_MAP_SIZE` +* Config: + + ```toml + [translation] + map-size = 10737418240 + ``` + ### Example Cluster Configuration A three node cluster running on different hosts could be minimally configured as follows: diff --git a/docs/data-model.md b/docs/data-model.md index ba7c12024..bababdd34 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust ### Field Type -Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`. +Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`. #### Set @@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00) #### Mutex Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field. + +#### Boolean + +A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values. diff --git a/docs/glossary.md b/docs/glossary.md index c39a0bf6f..fccbe0ae8 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -22,7 +22,7 @@ nav = [] Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). [Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. diff --git a/docs/query-language.md b/docs/query-language.md index 876cf69c6..9bc231e6f 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -236,6 +236,70 @@ Clear(10, stargazer=1) This represents removing the relationship between the user with id=1 and the repository with id=10. +#### ClearRow + +**Spec:** + +``` +ClearRow(=) +``` + +**Description:** + +`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns. + +**Result Type:** boolean + +A return value of `true` indicates that at least one column was toggled from 1 to 0. + +A return value of `false` indicates that all bits in the row were already 0 and nothing changed. + +**Examples:** + +Clear all bit in row 1 in the stargazer field: +```request +ClearRow(stargazer=1) +``` +```response +{"results":[true]} +``` + +This represents removing the relationship between the user with id=1 and all repositories. + +#### Store + +**Spec:** + +``` +Store(, =) +``` + +**Description:** + +`Store` writes the results of to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`. + +**Result Type:** boolean + +Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call. + +**Examples:** + +Store the contents of stargazer row 1 into stargazer row 2: +```request +Store(Row(stargazer=1), stargazer=2) +``` +```response +{"results":[true]} +``` + +Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20. +```request +Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20) +``` +```response +{"results":[true]} +``` + ### Read Operations #### Row diff --git a/docs/tutorials.md b/docs/tutorials.md index 040d85a1c..47344db4f 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \ {"success":true} ``` -Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. +Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system: ``` request diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 5aad595a5..00730328a 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package proto import ( @@ -812,30 +826,32 @@ func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { m.Node = &pilosa.Node{} - decodeIndexStatuses(pb.Indexes, m.Indexes) + m.Indexes = decodeIndexStatuses(pb.Indexes) m.Schema = &pilosa.Schema{} decodeSchema(pb.Schema, m.Schema) } -func decodeIndexStatuses(a []*internal.IndexStatus, m []*pilosa.IndexStatus) { - m = m[:0] +func decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.IndexStatus { + m := make([]*pilosa.IndexStatus, 0) for i := range a { m = append(m, &pilosa.IndexStatus{}) decodeIndexStatus(a[i], m[i]) } + return m } func decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name - decodeFieldStatuses(pb.Fields, m.Fields) + m.Fields = decodeFieldStatuses(pb.Fields) } -func decodeFieldStatuses(a []*internal.FieldStatus, m []*pilosa.FieldStatus) { - m = m[:0] +func decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.FieldStatus { + m := make([]*pilosa.FieldStatus, 0) for i := range a { m = append(m, &pilosa.FieldStatus{}) decodeFieldStatus(a[i], m[i]) } + return m } func decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index d687c88a3..7487eea70 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -88,13 +88,13 @@ func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Containe // this struct is added to prevent the closure locals from being escaped out to the heap type updater struct { key uint64 - n int + n int32 containerType byte mapped bool } func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { - a := updater{key, n, containerType, mapped} + a := updater{key, int32(n), containerType, mapped} btc.tree.Put(key, a.update) } diff --git a/executor.go b/executor.go index 89892e5c5..8cac5c034 100644 --- a/executor.go +++ b/executor.go @@ -81,20 +81,21 @@ func newExecutor(opts ...executorOption) *executor { } // Execute executes a PQL query. -func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { +func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { + resp := QueryResponse{} // Verify that an index is set. if index == "" { - return nil, ErrIndexRequired + return resp, ErrIndexRequired } idx := e.Holder.Index(index) if idx == nil { - return nil, ErrIndexNotFound + return resp, ErrIndexNotFound } // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { - return nil, ErrTooManyWrites + return resp, ErrTooManyWrites } // Default options. @@ -107,14 +108,48 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { for i := range q.Calls { if err := e.translateCall(index, idx, q.Calls[i]); err != nil { - return nil, err + return resp, err } } } results, err := e.execute(ctx, index, q, shards, opt) if err != nil { - return nil, err + return resp, err + } + + resp.Results = results + + // Fill column attributes if requested. + if opt.ColumnAttrs { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Row) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs) + if err != nil { + return resp, errors.Wrap(err, "reading column attrs") + } + + // Translate column attributes, if necessary. + if idx.Keys() { + for _, col := range columnAttrSets { + v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + + resp.ColumnAttrSets = columnAttrSets } // Translate response objects from ids to keys, if necessary. @@ -123,11 +158,35 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar for i := range results { results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) if err != nil { - return nil, err + return resp, err } } } - return results, nil + + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, errors.Wrap(err, "getting attrs") + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil } func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { @@ -184,6 +243,10 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeMax(ctx, index, c, shards, opt) case "Clear": return e.executeClearBit(ctx, index, c, opt) + case "ClearRow": + return e.executeClearRow(ctx, index, c, shards, opt) + case "Store": + return e.executeSetRow(ctx, index, c, shards, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) @@ -1055,8 +1118,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca rowID, rowOK, rowErr := c.UintArg(fieldName) if rowErr != nil { return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) - } - if !rowOK { + } else if !rowOK { return nil, fmt.Errorf("Row() must specify %v", rowLabel) } @@ -1429,7 +1491,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } -// executeClearBitField executes a Clear() call for a single view. +// executeClearBitField executes a Clear() call for a field. func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { shard := colID / ShardWidth ret := false @@ -1459,9 +1521,170 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } +// executeClearRow executes a ClearRow() call. +func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + // Ensure the field type supports ClearRow(). + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("ClearRow() argument required: field") + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + switch field.Type() { + case FieldTypeSet, FieldTypeTime, FieldTypeMutex, FieldTypeBool: + // These field types support ClearRow(). + default: + return false, fmt.Errorf("ClearRow() is not supported on %s field types", field.Type()) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeClearRowShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + val := v.(bool) + if prev == nil { + return val + } + return val || prev.(bool) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + return result.(bool), err +} + +// executeClearRowShard executes a ClearRow() call for a single shard. +func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("ClearRow() argument required: field") + } + + // Read fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading ClearRow() row: %v", err) + } else if !ok { + return false, fmt.Errorf("ClearRow() row argument '%v' required", rowLabel) + } + + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + // Remove the row from all views. + changed := false + for _, view := range field.views() { + fragment := e.Holder.fragment(index, fieldName, view.name, shard) + if fragment == nil { + continue + } + cleared, err := fragment.clearRow(rowID) + if err != nil { + return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard) + } + changed = changed || cleared + } + + return changed, nil +} + +// executeSetRow executes a SetRow() call. +func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + // Ensure the field type supports SetRow(). + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + if field.Type() != FieldTypeSet { + return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSetRowShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + val := v.(bool) + if prev == nil { + return val + } + return val || prev.(bool) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + return result.(bool), err +} + +// executeSetRowShard executes a SetRow() call for a single shard. +func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + + // Read fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading SetRow() row: %v", err) + } else if !ok { + return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel) + } + + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + // Retrieve source row. + var src *Row + if len(c.Children) == 1 { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return false, errors.Wrap(err, "getting source row") + } + src = row + } else { + return false, errors.New("SetRow() requires a source row") + } + + // Set the row on the standard view. + changed := false + fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) + if fragment == nil { + // Since the destination fragment doesn't exist, create one. + view, err := field.createViewIfNotExists(viewStandard) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + fragment, err = view.createFragmentIfNotExists(shard) + if err != nil { + return false, errors.Wrapf(err, "creating fragment: %d", shard) + } + } + set, err := fragment.setRow(src, rowID) + if err != nil { + return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) + } + changed = changed || set + + return changed, nil +} + // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { - // Read colID. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { @@ -1493,8 +1716,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } } + // Int field. if f.Type() == FieldTypeInt { - // Read remaining fields using labels. + // Read row value. rowVal, ok, err := c.IntArg(fieldName) if err != nil { return false, fmt.Errorf("reading Set() row: %v", err) @@ -1503,27 +1727,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) - } else { - // Read remaining fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) - if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) - } - timestamp = &t - } - - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } + + // Read row ID. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } // executeSetBitField executes a Set() call for a specific field. @@ -1954,21 +2178,22 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { var colKey, rowKey, fieldName string - if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + switch c.Name { + case "Set", "Clear", "Row", "Range", "SetColumnAttrs": // Positional args in new PQL syntax require special handling here. colKey = "_" + columnLabel fieldName, _ = c.FieldArg() rowKey = fieldName - } else if c.Name == "SetRowAttrs" { + case "SetRowAttrs": // Positional args in new PQL syntax require special handling here. rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") - } else if c.Name == "Rows" { + case "Rows": fieldName = callArgString(c, "field") rowKey = "previous" - } else if c.Name == "GroupBy" { + case "GroupBy": return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy") - } else { + default: colKey = "col" fieldName = callArgString(c, "field") rowKey = "row" @@ -2001,7 +2226,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { // will raise an error downstream when it's used. return nil } - if field.keys() { + + // Bool field keys do not use the translator because there + // are only two possible values. Instead, they are handled + // directly. + if field.Type() == FieldTypeBool { + boolVal, err := callArgBool(c, rowKey) + if err != nil { + return errors.Wrap(err, "getting bool key") + } + rowID := falseRowID + if boolVal { + rowID = trueRowID + } + c.Args[rowKey] = rowID + } else if field.keys() { if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { return errors.New("row value must be a string when field 'keys' option enabled") } @@ -2268,6 +2507,18 @@ func (vc *ValCount) larger(other ValCount) ValCount { } } +func callArgBool(call *pql.Call, key string) (bool, error) { + value, ok := call.Args[key] + if !ok { + return false, errors.New("missing bool argument") + } + b, ok := value.(bool) + if !ok { + return false, fmt.Errorf("invalid bool argument type: %T", value) + } + return b, nil +} + func callArgString(call *pql.Call, key string) string { value, ok := call.Args[key] if !ok { diff --git a/executor_test.go b/executor_test.go index 78e6d8790..366a044f9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) } +// Ensure a set query can be executed on a bool field. +func TestExecutor_Execute_SetBool(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set a true bit. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Set the same bit to true again verify nothing changed. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if res.Results[0].(bool) { + t.Fatalf("expected column to be unchanged") + } + + // Set the same bit to false. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Ensure that the false row is set. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) { + t.Fatalf("unexpected colums: %+v", columns) + } + + // Ensure that the true row is empty. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected colums: %+v", columns) + } + }) + t.Run("Error", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set bool using a string value. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + // Set bool using an integer. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + }) +} + // Ensure old PQL syntax doesn't break anything too badly. func TestExecutor_Execute_OldPQL(t *testing.T) { c := test.MustRunCluster(t, 1) @@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - // Create felds. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { t.Fatal(err) @@ -948,6 +1020,58 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } +// Ensure a range query with keys can be executed. +func TestExecutor_Execute_Range_WithKeys(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create index. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } + + // Set columns. + cc := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) + + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00) + ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + t.Run("Standard", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f="foo")`}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) +} + // Ensure a Range(bsiGroup) query can be executed. func TestExecutor_Execute_BSIGroupRange(t *testing.T) { c := test.MustRunCluster(t, 1) @@ -1453,6 +1577,36 @@ func TestExecutor_QueryCall(t *testing.T) { } }) + t.Run("columnAttrsWithKeys", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("one-hundred", f="ten") + SetColumnAttrs("one-hundred", foo="bar") + `}); err != nil { + t.Fatal(err) + } + + targetColAttrSets := []*pilosa.ColumnAttrSet{ + {Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}}, + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f="ten"), columnAttrs=true)`}); err != nil { + t.Fatal(err) + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + t.Run("shards", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -1601,6 +1755,401 @@ func TestExecutor_Execute_Not(t *testing.T) { } } +// Ensure a row can be cleared. +func TestExecutor_Execute_ClearRow(t *testing.T) { + t.Run("Set", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Clear the row again and ensure we get a `false` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Mutex", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0)) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Clear the row again and ensure we get a `false` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Time", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + if err != nil { + t.Fatal(err) + } + + // Set columns. + cc := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + ` + rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)` + rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)` + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Int", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100)) + if err != nil { + t.Fatal(err) + } + + // Ensure that clearing a row raises an error. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil { + t.Fatal("expected clear row to return an error") + } + }) + t.Run("TopN", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + cc := ` + Set(2, f=1) + Set(3, f=1) + Set(4, f=1) + Set(5, f=1) + Set(6, f=1) + Set(7, f=1) + Set(8, f=1) + + Set(2, f=2) + Set(3, f=2) + Set(4, f=2) + Set(5, f=2) + Set(6, f=2) + Set(7, f=2) + + Set(2, f=3) + Set(3, f=3) + Set(4, f=3) + Set(5, f=3) + Set(6, f=3) + ` + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + if err := c[0].RecalculateCaches(); err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + // Check the TopN results. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 2, Count: 6}, + {ID: 3, Count: 5}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache). + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 3, Count: 5}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + }) +} + +// Ensure a row can be set. +func TestExecutor_Execute_SetRow(t *testing.T) { + t.Run("Set_NewRow", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + if _, err := index.CreateField("tmp", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_NoSource", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a row that does exist. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_ExistingDestination", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into an existing row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/field.go b/field.go index d038b606c..27de57d8f 100644 --- a/field.go +++ b/field.go @@ -52,6 +52,7 @@ const ( FieldTypeInt = "int" FieldTypeTime = "time" FieldTypeMutex = "mutex" + FieldTypeBool = "bool" ) // Field represents a container for views. @@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { } } +func OptFieldTypeBool() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeBool + return nil + } +} + // NewField returns a new instance of field. func NewField(path, index, name string, opts FieldOption) (*Field, error) { err := validateName(name) @@ -219,8 +230,17 @@ func (f *Field) AvailableShards() *roaring.Bitmap { return b } -// addRemoteAvailableShards merges the set of available shards into the current known set. -func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) { +// addRemoteAvailableShards merges the set of available shards into the current known set +// and saves the set to a file. +func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error { + f.mergeRemoteAvailableShards(b) + + // Save the updated bitmap to the data store. + return f.saveAvailableShards() +} + +// mergeRemoteAvailableShards merges the set of available shards into the current known set. +func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) { f.mu.Lock() defer f.mu.Unlock() f.remoteAvailableShards = f.remoteAvailableShards.Union(b) @@ -280,6 +300,10 @@ func (f *Field) Open() error { return errors.Wrap(err, "loading meta") } + if err := f.loadAvailableShards(); err != nil { + return errors.Wrap(err, "loading available shards") + } + // Apply the field options loaded from meta. if err := f.applyOptions(f.options); err != nil { return errors.Wrap(err, "applying options") @@ -441,6 +465,14 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Max = 0 f.options.TimeQuantum = "" f.options.Keys = opt.Keys + case FieldTypeBool: + f.options.Type = FieldTypeBool + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + f.options.Keys = false default: return errors.New("invalid field type") } @@ -448,6 +480,47 @@ func (f *Field) applyOptions(opt FieldOptions) error { return nil } +// loadAvailableShards reads remoteAvailableShards data for the field, if any. +func (f *Field) loadAvailableShards() error { + bm := roaring.NewBitmap() + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "reading available shards") + } else { + if err := bm.UnmarshalBinary(buf); err != nil { + return errors.Wrap(err, "unmarshaling") + } + } + + // Merge bitmap from file into field. + f.mergeRemoteAvailableShards(bm) + + return nil +} + +// saveAvailableShards writes remoteAvailableShards data for the field. +func (f *Field) saveAvailableShards() error { + // Open or create file. + file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return errors.Wrap(err, "opening available shards file") + } + + f.mu.RLock() + defer f.mu.RUnlock() + + // Write available shards to file. + if _, err := f.remoteAvailableShards.WriteTo(file); err != nil { + return errors.Wrap(err, "writing bitmap to buffer") + } + + return nil +} + // Close closes the field and its views. func (f *Field) Close() error { f.mu.Lock() @@ -681,6 +754,10 @@ func (f *Field) deleteView(name string) error { } // Row returns a row of the standard view. +// It seems this method is only being used by the test +// package, and the fact that it's only allowed on +// `set` fields is odd. This may be considered for +// deprecation in a future version. func (f *Field) Row(rowID uint64) (*Row, error) { if f.Type() != FieldTypeSet { return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) @@ -954,10 +1031,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.New("time quantum not set in field") } + fieldType := f.Type() + // Split import data by fragment. dataByFragment := make(map[importKey]importData) for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] + + // Bool-specific data validation. + if fieldType == FieldTypeBool && rowID > 1 { + return errors.New("bool field imports only support values 0 and 1") + } + var timestamp *time.Time if len(timestamps) > i { timestamp = timestamps[i] @@ -1191,6 +1276,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.CacheSize, o.Keys, }) + case FieldTypeBool: + return json.Marshal(struct { + Type string `json:"type"` + }{ + o.Type, + }) } return nil, errors.New("invalid field type") } diff --git a/field_internal_test.go b/field_internal_test.go index 2d37ea25c..d72a3fd6a 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/roaring" ) // Ensure a bsiGroup can adjust to its baseValue. @@ -341,3 +342,22 @@ func TestField_RowTime(t *testing.T) { } } + +func TestField_PersistAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.addRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) { + t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice()) + } + +} diff --git a/fragment.go b/fragment.go index a22d2e1e4..9210866e7 100644 --- a/fragment.go +++ b/fragment.go @@ -75,6 +75,10 @@ const ( // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. defaultFragmentMaxOpN = 2000 + + // Row ids used for boolean fields. + falseRowID = uint64(0) + trueRowID = uint64(1) ) // fragment represents the intersection of a field and shard in an index. @@ -351,11 +355,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row { } // Only use a subset of the containers. - // NOTE: The start & end ranges must be divisible by + // NOTE: The start & end ranges must be divisible by container width. data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. - // We Clone() data because otherwise row will contains pointers to containers in storage. + // We Clone() data because otherwise row will contain pointers to containers in storage. // This causes unexpected results when we cache the row and try to use it later. row := &Row{ segments: []rowSegment{{ @@ -390,12 +394,13 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { // handleMutex will clear an existing row and store the new row // in the vector. func (f *fragment) handleMutex(rowID, columnID uint64) error { - if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID { + if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil { + return errors.Wrap(err, "getting mutex vector data") + } else if found && existingRowID != rowID { if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil { return errors.Wrap(err, "clearing mutex value") } } - f.mutexVector.Set(columnID, rowID) return nil } @@ -489,6 +494,95 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return changed, nil } +// setRow replaces an existing row (specified by rowID) with the given +// Row. This updates both the on-disk storage and the in-cache bitmap. +func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.unprotectedSetRow(row, rowID) +} + +func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) { + // TODO: In order to return `changed`, we need to first compare + // the existing row with the given row. Determine if the overhead + // of this is worth having `changed`. + // For now we will assume changed is always true. + changed = true + + // First container of the row in storage. + headContainerKey := rowID << shardVsContainerExponent + + // Remove every existing container in the row. + for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { + f.storage.Containers.Remove(headContainerKey + i) + } + + // From the given row, get the rowSegment for this shard. + seg := row.segment(f.shard) + if seg == nil { + return changed, nil + } + + // Put each container from rowSegment to fragment storage. + citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) + for citer.Next() { + k, c := citer.Value() + f.storage.Containers.Put(headContainerKey+(k%(1< 1 { + return 0, false, errors.New("found multiple row values for column") + } else if len(rows) == 1 { + return rows[0], true, nil } - return 0, false + return 0, false, nil } -// Set is not used for rowsVector. -func (v *rowsVector) Set(colID, rowID uint64) {} - // rowToKey converts a Pilosa row ID to the key of the container which starts // that row in the bitmap which represents this entire fragment. A fragment is // all the rows within a shard within a field concatenated together. func rowToKey(rowID uint64) (key uint64) { return rowID * (ShardWidth / containerWidth) } + +// boolVector implements the vector interface by looking +// at data in rows 0 and 1. +type boolVector struct { + f *fragment +} + +// newBoolVector returns a boolVector for a given fragment. +func newBoolVector(f *fragment) *boolVector { + return &boolVector{ + f: f, + } +} + +// Get returns the rowID associated to the given colID. +// Additionally, it returns true if a value was found, +// otherwise it returns false. +func (v *boolVector) Get(colID uint64) (uint64, bool, error) { + rows := v.f.rows(0, filterColumn(colID)) + if len(rows) > 1 { + return 0, false, errors.New("found multiple row values for column") + } else if len(rows) == 1 { + switch rows[0] { + case falseRowID, trueRowID: + return rows[0], true, nil + default: + return 0, false, errors.New("found non-boolean value") + } + } + return 0, false, nil +} diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 36651fee0..59d15ef54 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -95,6 +95,81 @@ func TestFragment_ClearBit(t *testing.T) { } } +// Ensure a fragment can clear a row. +func TestFragment_ClearRow(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + // Set and then clear bits on the fragment. + if _, err := f.setBit(1000, 1); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(1000, 65536); err != nil { + t.Fatal(err) + } else if _, err := f.unprotectedClearRow(1000); err != nil { + t.Fatal(err) + } + + // Verify count on row. + if n := f.row(1000).Count(); n != 0 { + t.Fatalf("unexpected count: %d", n) + } + + // Close and reopen the fragment & verify the data. + if err := f.reopen(); err != nil { + t.Fatal(err) + } else if n := f.row(1000).Count(); n != 0 { + t.Fatalf("unexpected count (reopen): %d", n) + } +} + +// Ensure a fragment can set a row. +func TestFragment_SetRow(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 7, "") + defer f.Close() + + rowID := uint64(1000) + + // Set bits on the fragment. + if _, err := f.setBit(rowID, 8000001); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(rowID, 8065536); err != nil { + t.Fatal(err) + } + + // Verify data on row. + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) { + t.Fatalf("unexpected columns: %+v", cols) + } + // Verify count on row. + if n := f.row(rowID).Count(); n != 2 { + t.Fatalf("unexpected count: %d", n) + } + + // Set row (overwrite existing data). + row := NewRow(8000002, 8065537, 8131074) + if changed, err := f.unprotectedSetRow(row, rowID); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatalf("expected changed value: %v", changed) + } + + // Verify data on row. + if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) { + t.Fatalf("unexpected columns after set row: %+v", cols) + } + // Verify count on row. + if n := f.row(rowID).Count(); n != 3 { + t.Fatalf("unexpected count after set row: %d", n) + } + + // Close and reopen the fragment & verify the data. + if err := f.reopen(); err != nil { + t.Fatal(err) + } else if n := f.row(rowID).Count(); n != 3 { + t.Fatalf("unexpected count (reopen): %d", n) + } +} + // Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { @@ -1178,6 +1253,146 @@ func TestFragment_SetMutex(t *testing.T) { } } +// Ensure a fragment can import mutually exclusive values. +func TestFragment_ImportMutex(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + exp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 2, 2}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 1: {0, 2, 3}, + 2: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {8}, + }, + }, + { + []uint64{1, 2, 3}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 1: {}, + 2: {}, + 3: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + // Check for expected results. + for k, v := range test.exp { + cols := f.row(k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + +// Ensure a fragment can import bool values. +func TestFragment_ImportBool(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + exp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1, 1, 1, 1}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 0: {}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 0: {0, 2, 3}, + 1: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 0: {8}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 1, 2}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 0: {}, + 1: {}, // This isn't {8} because fragment doesn't validate bool values. + 2: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + // Check for expected results. + for k, v := range test.exp { + cols := f.row(k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func BenchmarkFragment_Snapshot(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -1302,6 +1517,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st return frag } +// mustOpenBoolFragment returns a new instance of Fragment for a bool field. +func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment { + frag := mustOpenFragment(index, field, view, shard, cacheType) + frag.mutexVector = newBoolVector(frag) + return frag +} + // Reopen closes the fragment and reopens it as a new instance. func (f *fragment) reopen() error { if err := f.Close(); err != nil { diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 8cdd77778..290e1278c 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -15,6 +15,8 @@ package gopsutil import ( + "runtime" + "github.com/pilosa/pilosa" "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/mem" @@ -109,6 +111,11 @@ func (s *systemInfo) KernelVersion() (string, error) { return host.KernelVersion() } +// CPUArch returns the CPU architecture, such as amd64 +func (s *systemInfo) CPUArch() string { + return runtime.GOARCH +} + // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. func NewSystemInfo() *systemInfo { return &systemInfo{} diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go index 0f76b62da..28d09e66a 100644 --- a/gopsutil/systeminfo_test.go +++ b/gopsutil/systeminfo_test.go @@ -15,7 +15,6 @@ package gopsutil_test import ( - "log" "testing" "github.com/pilosa/pilosa" @@ -25,15 +24,6 @@ import ( func TestSystemInfo(t *testing.T) { var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo() - // Uptime()(uint64, error) - // Platform()(string, error) - // Family()(string, error) - // OSVersion()(string, error) - // KernelVersion()(string, error) - // MemFree()(uint64, error) - // MemTotal()(uint64, error) - // MemUsed()(uint64, error) - // uptime, err := systemInfo.Uptime() if err != nil || uptime == 0 { t.Fatalf("Error collecting uptime (error: %v)", err) @@ -70,8 +60,12 @@ func TestSystemInfo(t *testing.T) { } memtotal, err := systemInfo.MemTotal() - log.Println(memtotal) if err != nil { t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err) } + + cpuArch := systemInfo.CPUArch() + if cpuArch == "" { + t.Fatalf("Error getting CPU arch.") + } } diff --git a/http/handler.go b/http/handler.go index bc108568a..cb242475c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -170,13 +170,34 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") - h.validators["GetShardMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["Home"] = queryValidationSpecRequired() + h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired() + h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired() + h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired() h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetIndexes"] = queryValidationSpecRequired() + h.validators["GetIndex"] = queryValidationSpecRequired() + h.validators["PostIndex"] = queryValidationSpecRequired() + h.validators["DeleteIndex"] = queryValidationSpecRequired() + h.validators["PostField"] = queryValidationSpecRequired() + h.validators["DeleteField"] = queryValidationSpecRequired() + h.validators["PostImport"] = queryValidationSpecRequired() + h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetInfo"] = queryValidationSpecRequired() + h.validators["RecalculateCaches"] = queryValidationSpecRequired() + h.validators["GetSchema"] = queryValidationSpecRequired() + h.validators["GetStatus"] = queryValidationSpecRequired() + h.validators["GetVersion"] = queryValidationSpecRequired() + h.validators["PostClusterMessage"] = queryValidationSpecRequired() + h.validators["GetFragmentBlockData"] = queryValidationSpecRequired() h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") + h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired() + h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired() + h.validators["GetNodes"] = queryValidationSpecRequired() + h.validators["GetShardMax"] = queryValidationSpecRequired() + h.validators["GetTranslateData"] = queryValidationSpecRequired("offset") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -203,40 +224,40 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET") - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home") + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex") //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") - router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") // /internal endpoints are for internal use only; they may change at any time. // DO NOT rely on these for external applications! - router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage") + router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData") router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff") + router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") - router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET") + router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: @@ -521,7 +542,12 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error { return err } // Unmarshal expected values. - var _p _postIndexRequest + _p := _postIndexRequest{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }, + } if err := json.Unmarshal(b, &_p); err != nil { return errors.Wrap(err, "unmarshalling expected values") } @@ -597,7 +623,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp := successResponse{} // Decode request. - var req postIndexRequest + req := postIndexRequest{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }, + } err := json.NewDecoder(r.Body).Decode(&req) if err != nil && err != io.EOF { resp.write(w, err) @@ -687,6 +718,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) case pilosa.FieldTypeMutex: fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize)) + case pilosa.FieldTypeBool: + fos = append(fos, pilosa.OptFieldTypeBool()) } if req.Options.Keys != nil { if *req.Options.Keys { @@ -778,6 +811,20 @@ func (o *fieldOptions) validate() error { } else if o.TimeQuantum != nil { return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } default: return errors.Errorf("invalid field type: %s", o.Type) } @@ -1457,11 +1504,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } + resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body) - resp := &pilosa.ImportResponse{} if err != nil { resp.Err = err.Error() + if _, ok := err.(pilosa.BadRequestError); ok { + w.WriteHeader(http.StatusBadRequest) + } else { + w.WriteHeader(http.StatusInternalServerError) + } } // Marshal response object. buf, err := h.api.Serializer.Marshal(resp) diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index b0852a79d..7ca9c0401 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -31,8 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, - {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}}, + {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, + {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, @@ -53,7 +54,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { if test.err == "" { if !reflect.DeepEqual(*actual, test.expected) { - t.Errorf("expected: %v, but got: %v", test.expected, *actual) + t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json) } } diff --git a/index.go b/index.go index 0992ab213..297c02f20 100644 --- a/index.go +++ b/index.go @@ -69,9 +69,10 @@ func NewIndex(path, name string) (*Index, error) { newAttrStore: newNopAttrStore, columnAttrs: nopStore, - broadcaster: NopBroadcaster, - Stats: NopStatsClient, - logger: NopLogger, + broadcaster: NopBroadcaster, + Stats: NopStatsClient, + logger: NopLogger, + trackExistence: true, }, nil } diff --git a/pilosa.go b/pilosa.go index 96b06d9e6..d2fdf8144 100644 --- a/pilosa.go +++ b/pilosa.go @@ -119,7 +119,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { - ID uint64 `json:"id"` + ID uint64 `json:"id,omitempty"` Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } diff --git a/pql/ast.go b/pql/ast.go index cd483e030..0e2ff6aef 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) { return "", fmt.Errorf("No field argument specified") } +// BoolArg is for reading the value at key from call.Args as a bool. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a bool. An error is +// returned if the value is not a bool. +func (c *Call) BoolArg(key string) (bool, bool, error) { + val, ok := c.Args[key] + if !ok { + return false, false, nil + } + switch tval := val.(type) { + case bool: + return tval, true, nil + default: + return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval) + } +} + // UintArg is for reading the value at key from call.Args as a uint64. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a uint64 or an int64 and diff --git a/pql/pql.peg b/pql/pql.peg index 33896f8ad..9e0087046 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -10,6 +10,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()} / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} + / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} + / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 21166d0ca..2d5877fbb 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -62,11 +62,11 @@ const ( ruleAction9 ruleAction10 ruleAction11 - rulePegText ruleAction12 ruleAction13 ruleAction14 ruleAction15 + rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -99,6 +99,10 @@ const ( ruleAction45 ruleAction46 ruleAction47 + ruleAction48 + ruleAction49 + ruleAction50 + ruleAction51 ) var rul3s = [...]string{ @@ -149,11 +153,11 @@ var rul3s = [...]string{ "Action9", "Action10", "Action11", - "PegText", "Action12", "Action13", "Action14", "Action15", + "PegText", "Action16", "Action17", "Action18", @@ -186,6 +190,10 @@ var rul3s = [...]string{ "Action45", "Action46", "Action47", + "Action48", + "Action49", + "Action50", + "Action51", } type token32 struct { @@ -302,7 +310,7 @@ type PQL struct { Buffer string buffer []rune - rules [84]func() bool + rules [88]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -411,84 +419,92 @@ func (p *PQL) Execute() { case ruleAction7: p.endCall() case ruleAction8: - p.startCall("TopN") + p.startCall("ClearRow") case ruleAction9: p.endCall() case ruleAction10: - p.startCall("Range") + p.startCall("Store") case ruleAction11: p.endCall() case ruleAction12: - p.startCall(buffer[begin:end]) + p.startCall("TopN") case ruleAction13: p.endCall() case ruleAction14: - p.addBTWN() + p.startCall("Range") case ruleAction15: - p.addLTE() + p.endCall() case ruleAction16: - p.addGTE() + p.startCall(buffer[begin:end]) case ruleAction17: - p.addEQ() + p.endCall() case ruleAction18: - p.addNEQ() + p.addBTWN() case ruleAction19: - p.addLT() + p.addLTE() case ruleAction20: - p.addGT() + p.addGTE() case ruleAction21: - p.startConditional() + p.addEQ() case ruleAction22: - p.endConditional() + p.addNEQ() case ruleAction23: - p.condAdd(buffer[begin:end]) + p.addLT() case ruleAction24: - p.condAdd(buffer[begin:end]) + p.addGT() case ruleAction25: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction26: - p.addPosStr("_start", buffer[begin:end]) + p.endConditional() case ruleAction27: - p.addPosStr("_end", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction28: - p.startList() + p.condAdd(buffer[begin:end]) case ruleAction29: - p.endList() + p.condAdd(buffer[begin:end]) case ruleAction30: - p.addVal(nil) + p.addPosStr("_start", buffer[begin:end]) case ruleAction31: - p.addVal(true) + p.addPosStr("_end", buffer[begin:end]) case ruleAction32: - p.addVal(false) + p.startList() case ruleAction33: - p.addNumVal(buffer[begin:end]) + p.endList() case ruleAction34: - p.addNumVal(buffer[begin:end]) + p.addVal(nil) case ruleAction35: - p.addVal(buffer[begin:end]) + p.addVal(true) case ruleAction36: - p.addVal(buffer[begin:end]) + p.addVal(false) case ruleAction37: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction38: - p.addField(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction39: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction40: - p.addPosNum("_row", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction41: - p.addPosNum("_col", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction42: - p.addPosStr("_col", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction43: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction44: p.addPosNum("_row", buffer[begin:end]) case ruleAction45: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_col", buffer[begin:end]) case ruleAction46: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction47: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction48: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction49: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -601,7 +617,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'a' 'n' 'g' 'e' Action14 open (timerange / conditional / arg) close Action15) / ( Action16 open allargs comma? close Action17))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -650,7 +666,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction47, position) + add(ruleAction51, position) } add(ruletimestamp, position12) } @@ -736,7 +752,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction44, position) + add(ruleAction48, position) } goto l19 l20: @@ -757,7 +773,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction45, position) + add(ruleAction49, position) } goto l19 l23: @@ -778,7 +794,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction46, position) + add(ruleAction50, position) } } l19: @@ -923,7 +939,27 @@ func (p *PQL) Init() { goto l7 l32: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('T') { + if buffer[position] != rune('C') { + goto l35 + } + position++ + if buffer[position] != rune('l') { + goto l35 + } + position++ + if buffer[position] != rune('e') { + goto l35 + } + position++ + if buffer[position] != rune('a') { + goto l35 + } + position++ + if buffer[position] != rune('r') { + goto l35 + } + position++ + if buffer[position] != rune('R') { goto l35 } position++ @@ -931,11 +967,7 @@ func (p *PQL) Init() { goto l35 } position++ - if buffer[position] != rune('p') { - goto l35 - } - position++ - if buffer[position] != rune('N') { + if buffer[position] != rune('w') { goto l35 } position++ @@ -945,22 +977,9 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l35 } - if !_rules[ruleposfield]() { + if !_rules[rulearg]() { goto l35 } - { - position37, tokenIndex37 := position, tokenIndex - if !_rules[rulecomma]() { - goto l37 - } - if !_rules[ruleallargs]() { - goto l37 - } - goto l38 - l37: - position, tokenIndex = position37, tokenIndex37 - } - l38: if !_rules[ruleclose]() { goto l35 } @@ -970,193 +989,284 @@ func (p *PQL) Init() { goto l7 l35: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l40 + if buffer[position] != rune('S') { + goto l38 } position++ - if buffer[position] != rune('a') { - goto l40 + if buffer[position] != rune('t') { + goto l38 } position++ - if buffer[position] != rune('n') { - goto l40 + if buffer[position] != rune('o') { + goto l38 } position++ - if buffer[position] != rune('g') { - goto l40 + if buffer[position] != rune('r') { + goto l38 } position++ if buffer[position] != rune('e') { - goto l40 + goto l38 } position++ { add(ruleAction10, position) } if !_rules[ruleopen]() { - goto l40 + goto l38 } - { - position42, tokenIndex42 := position, tokenIndex - { - position44 := position - if !_rules[rulefield]() { - goto l43 - } - if !_rules[rulesp]() { - goto l43 - } - if buffer[position] != rune('=') { - goto l43 - } - position++ - if !_rules[rulesp]() { - goto l43 - } - if !_rules[rulevalue]() { - goto l43 - } - if !_rules[rulecomma]() { - goto l43 - } - { - position45 := position - if !_rules[ruletimestampfmt]() { - goto l43 - } - add(rulePegText, position45) - } - { - add(ruleAction26, position) - } - if !_rules[rulecomma]() { - goto l43 - } - { - position47 := position - if !_rules[ruletimestampfmt]() { - goto l43 - } - add(rulePegText, position47) - } - { - add(ruleAction27, position) - } - add(ruletimerange, position44) - } - goto l42 - l43: - position, tokenIndex = position42, tokenIndex42 - { - position50 := position - { - add(ruleAction21, position) - } - if !_rules[rulecondint]() { - goto l49 - } - if !_rules[rulecondLT]() { - goto l49 - } - { - position52 := position - { - position53 := position - if !_rules[rulefieldExpr]() { - goto l49 - } - add(rulePegText, position53) - } - if !_rules[rulesp]() { - goto l49 - } - { - add(ruleAction25, position) - } - add(rulecondfield, position52) - } - if !_rules[rulecondLT]() { - goto l49 - } - if !_rules[rulecondint]() { - goto l49 - } - { - add(ruleAction22, position) - } - add(ruleconditional, position50) - } - goto l42 - l49: - position, tokenIndex = position42, tokenIndex42 - if !_rules[rulearg]() { - goto l40 - } + if !_rules[ruleCall]() { + goto l38 + } + if !_rules[rulecomma]() { + goto l38 + } + if !_rules[rulearg]() { + goto l38 } - l42: if !_rules[ruleclose]() { - goto l40 + goto l38 } { add(ruleAction11, position) } goto l7 - l40: + l38: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('T') { + goto l41 + } + position++ + if buffer[position] != rune('o') { + goto l41 + } + position++ + if buffer[position] != rune('p') { + goto l41 + } + position++ + if buffer[position] != rune('N') { + goto l41 + } + position++ + { + add(ruleAction12, position) + } + if !_rules[ruleopen]() { + goto l41 + } + if !_rules[ruleposfield]() { + goto l41 + } + { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulecomma]() { + goto l43 + } + if !_rules[ruleallargs]() { + goto l43 + } + goto l44 + l43: + position, tokenIndex = position43, tokenIndex43 + } + l44: + if !_rules[ruleclose]() { + goto l41 + } + { + add(ruleAction13, position) + } + goto l7 + l41: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l46 + } + position++ + if buffer[position] != rune('a') { + goto l46 + } + position++ + if buffer[position] != rune('n') { + goto l46 + } + position++ + if buffer[position] != rune('g') { + goto l46 + } + position++ + if buffer[position] != rune('e') { + goto l46 + } + position++ + { + add(ruleAction14, position) + } + if !_rules[ruleopen]() { + goto l46 + } + { + position48, tokenIndex48 := position, tokenIndex + { + position50 := position + if !_rules[rulefield]() { + goto l49 + } + if !_rules[rulesp]() { + goto l49 + } + if buffer[position] != rune('=') { + goto l49 + } + position++ + if !_rules[rulesp]() { + goto l49 + } + if !_rules[rulevalue]() { + goto l49 + } + if !_rules[rulecomma]() { + goto l49 + } + { + position51 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position51) + } + { + add(ruleAction30, position) + } + if !_rules[rulecomma]() { + goto l49 + } + { + position53 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position53) + } + { + add(ruleAction31, position) + } + add(ruletimerange, position50) + } + goto l48 + l49: + position, tokenIndex = position48, tokenIndex48 + { + position56 := position + { + add(ruleAction25, position) + } + if !_rules[rulecondint]() { + goto l55 + } + if !_rules[rulecondLT]() { + goto l55 + } + { + position58 := position + { + position59 := position + if !_rules[rulefieldExpr]() { + goto l55 + } + add(rulePegText, position59) + } + if !_rules[rulesp]() { + goto l55 + } + { + add(ruleAction29, position) + } + add(rulecondfield, position58) + } + if !_rules[rulecondLT]() { + goto l55 + } + if !_rules[rulecondint]() { + goto l55 + } + { + add(ruleAction26, position) + } + add(ruleconditional, position56) + } + goto l48 + l55: + position, tokenIndex = position48, tokenIndex48 + if !_rules[rulearg]() { + goto l46 + } + } + l48: + if !_rules[ruleclose]() { + goto l46 + } + { + add(ruleAction15, position) + } + goto l7 + l46: position, tokenIndex = position7, tokenIndex7 { - position57 := position + position63 := position { - position58 := position + position64 := position { - position59, tokenIndex59 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 + goto l66 } position++ - goto l59 - l60: - position, tokenIndex = position59, tokenIndex59 + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l59: - l61: + l65: + l67: { - position62, tokenIndex62 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex { - position63, tokenIndex63 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l64 + goto l70 } position++ - goto l63 - l64: - position, tokenIndex = position63, tokenIndex63 + goto l69 + l70: + position, tokenIndex = position69, tokenIndex69 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l65 + goto l71 } position++ - goto l63 - l65: - position, tokenIndex = position63, tokenIndex63 + goto l69 + l71: + position, tokenIndex = position69, tokenIndex69 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l62 + goto l68 } position++ } - l63: - goto l61 - l62: - position, tokenIndex = position62, tokenIndex62 + l69: + goto l67 + l68: + position, tokenIndex = position68, tokenIndex68 } - add(ruleIDENT, position58) + add(ruleIDENT, position64) } - add(rulePegText, position57) + add(rulePegText, position63) } { - add(ruleAction12, position) + add(ruleAction16, position) } if !_rules[ruleopen]() { goto l5 @@ -1165,20 +1275,20 @@ func (p *PQL) Init() { goto l5 } { - position67, tokenIndex67 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if !_rules[rulecomma]() { - goto l67 + goto l73 } - goto l68 - l67: - position, tokenIndex = position67, tokenIndex67 + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 } - l68: + l74: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction13, position) + add(ruleAction17, position) } } l7: @@ -1191,1487 +1301,1487 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position70, tokenIndex70 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position71 := position + position77 := position { - position72, tokenIndex72 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex if !_rules[ruleCall]() { - goto l73 + goto l79 } - l74: + l80: { - position75, tokenIndex75 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex if !_rules[rulecomma]() { - goto l75 + goto l81 } if !_rules[ruleCall]() { - goto l75 + goto l81 } - goto l74 - l75: - position, tokenIndex = position75, tokenIndex75 + goto l80 + l81: + position, tokenIndex = position81, tokenIndex81 } { - position76, tokenIndex76 := position, tokenIndex + position82, tokenIndex82 := position, tokenIndex if !_rules[rulecomma]() { - goto l76 + goto l82 } if !_rules[ruleargs]() { - goto l76 + goto l82 } - goto l77 - l76: - position, tokenIndex = position76, tokenIndex76 + goto l83 + l82: + position, tokenIndex = position82, tokenIndex82 } - l77: - goto l72 - l73: - position, tokenIndex = position72, tokenIndex72 + l83: + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 if !_rules[ruleargs]() { - goto l78 + goto l84 } - goto l72 - l78: - position, tokenIndex = position72, tokenIndex72 + goto l78 + l84: + position, tokenIndex = position78, tokenIndex78 if !_rules[rulesp]() { - goto l70 + goto l76 } } - l72: - add(ruleallargs, position71) + l78: + add(ruleallargs, position77) } return true - l70: - position, tokenIndex = position70, tokenIndex70 + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position79, tokenIndex79 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex { - position80 := position + position86 := position if !_rules[rulearg]() { - goto l79 + goto l85 } { - position81, tokenIndex81 := position, tokenIndex + position87, tokenIndex87 := position, tokenIndex if !_rules[rulecomma]() { - goto l81 + goto l87 } if !_rules[ruleargs]() { - goto l81 + goto l87 } - goto l82 - l81: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l87: + position, tokenIndex = position87, tokenIndex87 } - l82: + l88: if !_rules[rulesp]() { - goto l79 + goto l85 } - add(ruleargs, position80) + add(ruleargs, position86) } return true - l79: - position, tokenIndex = position79, tokenIndex79 + l85: + position, tokenIndex = position85, tokenIndex85 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position83, tokenIndex83 := position, tokenIndex + position89, tokenIndex89 := position, tokenIndex { - position84 := position + position90 := position { - position85, tokenIndex85 := position, tokenIndex + position91, tokenIndex91 := position, tokenIndex if !_rules[rulefield]() { - goto l86 + goto l92 } if !_rules[rulesp]() { - goto l86 + goto l92 } if buffer[position] != rune('=') { - goto l86 + goto l92 } position++ if !_rules[rulesp]() { - goto l86 + goto l92 } if !_rules[rulevalue]() { - goto l86 + goto l92 } - goto l85 - l86: - position, tokenIndex = position85, tokenIndex85 + goto l91 + l92: + position, tokenIndex = position91, tokenIndex91 if !_rules[rulefield]() { - goto l83 + goto l89 } if !_rules[rulesp]() { - goto l83 + goto l89 } { - position87 := position + position93 := position { - position88, tokenIndex88 := position, tokenIndex + position94, tokenIndex94 := position, tokenIndex if buffer[position] != rune('>') { - goto l89 - } - position++ - if buffer[position] != rune('<') { - goto l89 - } - position++ - { - add(ruleAction14, position) - } - goto l88 - l89: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('<') { - goto l91 - } - position++ - if buffer[position] != rune('=') { - goto l91 - } - position++ - { - add(ruleAction15, position) - } - goto l88 - l91: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('>') { - goto l93 - } - position++ - if buffer[position] != rune('=') { - goto l93 - } - position++ - { - add(ruleAction16, position) - } - goto l88 - l93: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('=') { goto l95 } position++ - if buffer[position] != rune('=') { + if buffer[position] != rune('<') { goto l95 } position++ - { - add(ruleAction17, position) - } - goto l88 - l95: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('!') { - goto l97 - } - position++ - if buffer[position] != rune('=') { - goto l97 - } - position++ { add(ruleAction18, position) } - goto l88 - l97: - position, tokenIndex = position88, tokenIndex88 + goto l94 + l95: + position, tokenIndex = position94, tokenIndex94 if buffer[position] != rune('<') { - goto l99 + goto l97 + } + position++ + if buffer[position] != rune('=') { + goto l97 } position++ { add(ruleAction19, position) } - goto l88 - l99: - position, tokenIndex = position88, tokenIndex88 + goto l94 + l97: + position, tokenIndex = position94, tokenIndex94 if buffer[position] != rune('>') { - goto l83 + goto l99 + } + position++ + if buffer[position] != rune('=') { + goto l99 } position++ { add(ruleAction20, position) } + goto l94 + l99: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('=') { + goto l101 + } + position++ + if buffer[position] != rune('=') { + goto l101 + } + position++ + { + add(ruleAction21, position) + } + goto l94 + l101: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('!') { + goto l103 + } + position++ + if buffer[position] != rune('=') { + goto l103 + } + position++ + { + add(ruleAction22, position) + } + goto l94 + l103: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('<') { + goto l105 + } + position++ + { + add(ruleAction23, position) + } + goto l94 + l105: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('>') { + goto l89 + } + position++ + { + add(ruleAction24, position) + } } - l88: - add(ruleCOND, position87) + l94: + add(ruleCOND, position93) } if !_rules[rulesp]() { - goto l83 + goto l89 } if !_rules[rulevalue]() { - goto l83 + goto l89 } } - l85: - add(rulearg, position84) + l91: + add(rulearg, position90) } return true - l83: - position, tokenIndex = position83, tokenIndex83 + l89: + position, tokenIndex = position89, tokenIndex89 return false }, - /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ + /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ nil, - /* 6 conditional <- <(Action21 condint condLT condfield condLT condint Action22)> */ + /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ func() bool { - position104, tokenIndex104 := position, tokenIndex + position110, tokenIndex110 := position, tokenIndex { - position105 := position + position111 := position { - position106 := position + position112 := position { - position107, tokenIndex107 := position, tokenIndex + position113, tokenIndex113 := position, tokenIndex { - position109, tokenIndex109 := position, tokenIndex + position115, tokenIndex115 := position, tokenIndex if buffer[position] != rune('-') { - goto l109 + goto l115 } position++ - goto l110 - l109: - position, tokenIndex = position109, tokenIndex109 + goto l116 + l115: + position, tokenIndex = position115, tokenIndex115 } - l110: + l116: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l108 - } - position++ - l111: - { - position112, tokenIndex112 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l112 - } - position++ - goto l111 - l112: - position, tokenIndex = position112, tokenIndex112 - } - goto l107 - l108: - position, tokenIndex = position107, tokenIndex107 - if buffer[position] != rune('0') { - goto l104 - } - position++ - } - l107: - add(rulePegText, position106) - } - if !_rules[rulesp]() { - goto l104 - } - { - add(ruleAction23, position) - } - add(rulecondint, position105) - } - return true - l104: - position, tokenIndex = position104, tokenIndex104 - return false - }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ - func() bool { - position114, tokenIndex114 := position, tokenIndex - { - position115 := position - { - position116 := position - { - position117, tokenIndex117 := position, tokenIndex - if buffer[position] != rune('<') { - goto l118 - } - position++ - if buffer[position] != rune('=') { - goto l118 - } - position++ - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 - if buffer[position] != rune('<') { goto l114 } position++ + l117: + { + position118, tokenIndex118 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l118 + } + position++ + goto l117 + l118: + position, tokenIndex = position118, tokenIndex118 + } + goto l113 + l114: + position, tokenIndex = position113, tokenIndex113 + if buffer[position] != rune('0') { + goto l110 + } + position++ } - l117: - add(rulePegText, position116) + l113: + add(rulePegText, position112) } if !_rules[rulesp]() { - goto l114 + goto l110 } { - add(ruleAction24, position) + add(ruleAction27, position) } - add(rulecondLT, position115) + add(rulecondint, position111) } return true - l114: - position, tokenIndex = position114, tokenIndex114 + l110: + position, tokenIndex = position110, tokenIndex110 return false }, - /* 9 condfield <- <( sp Action25)> */ - nil, - /* 10 timerange <- <(field sp '=' sp value comma Action26 comma Action27)> */ - nil, - /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position122, tokenIndex122 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex { - position123 := position + position121 := position { - position124, tokenIndex124 := position, tokenIndex + position122 := position + { + position123, tokenIndex123 := position, tokenIndex + if buffer[position] != rune('<') { + goto l124 + } + position++ + if buffer[position] != rune('=') { + goto l124 + } + position++ + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('<') { + goto l120 + } + position++ + } + l123: + add(rulePegText, position122) + } + if !_rules[rulesp]() { + goto l120 + } + { + add(ruleAction28, position) + } + add(rulecondLT, position121) + } + return true + l120: + position, tokenIndex = position120, tokenIndex120 + return false + }, + /* 9 condfield <- <( sp Action29)> */ + nil, + /* 10 timerange <- <(field sp '=' sp value comma Action30 comma Action31)> */ + nil, + /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ + func() bool { + position128, tokenIndex128 := position, tokenIndex + { + position129 := position + { + position130, tokenIndex130 := position, tokenIndex if !_rules[ruleitem]() { - goto l125 + goto l131 } - goto l124 - l125: - position, tokenIndex = position124, tokenIndex124 - { - position126 := position - if buffer[position] != rune('[') { - goto l122 - } - position++ - if !_rules[rulesp]() { - goto l122 - } - add(rulelbrack, position126) - } - { - add(ruleAction28, position) - } - if !_rules[rulelist]() { - goto l122 - } - { - position128 := position - if !_rules[rulesp]() { - goto l122 - } - if buffer[position] != rune(']') { - goto l122 - } - position++ - if !_rules[rulesp]() { - goto l122 - } - add(rulerbrack, position128) - } - { - add(ruleAction29, position) - } - } - l124: - add(rulevalue, position123) - } - return true - l122: - position, tokenIndex = position122, tokenIndex122 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position130, tokenIndex130 := position, tokenIndex - { - position131 := position - if !_rules[ruleitem]() { goto l130 - } - { - position132, tokenIndex132 := position, tokenIndex - if !_rules[rulecomma]() { - goto l132 - } - if !_rules[rulelist]() { - goto l132 - } - goto l133 - l132: - position, tokenIndex = position132, tokenIndex132 - } - l133: - add(rulelist, position131) - } - return true - l130: - position, tokenIndex = position130, tokenIndex130 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ - func() bool { - position134, tokenIndex134 := position, tokenIndex - { - position135 := position - { - position136, tokenIndex136 := position, tokenIndex - if buffer[position] != rune('n') { - goto l137 - } - position++ - if buffer[position] != rune('u') { - goto l137 - } - position++ - if buffer[position] != rune('l') { - goto l137 - } - position++ - if buffer[position] != rune('l') { - goto l137 - } - position++ + l131: + position, tokenIndex = position130, tokenIndex130 { - position138, tokenIndex138 := position, tokenIndex - { - position139, tokenIndex139 := position, tokenIndex - if !_rules[rulecomma]() { - goto l140 - } - goto l139 - l140: - position, tokenIndex = position139, tokenIndex139 - if !_rules[rulesp]() { - goto l137 - } - if !_rules[ruleclose]() { - goto l137 - } + position132 := position + if buffer[position] != rune('[') { + goto l128 } - l139: - position, tokenIndex = position138, tokenIndex138 - } - { - add(ruleAction30, position) - } - goto l136 - l137: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('t') { - goto l142 - } - position++ - if buffer[position] != rune('r') { - goto l142 - } - position++ - if buffer[position] != rune('u') { - goto l142 - } - position++ - if buffer[position] != rune('e') { - goto l142 - } - position++ - { - position143, tokenIndex143 := position, tokenIndex - { - position144, tokenIndex144 := position, tokenIndex - if !_rules[rulecomma]() { - goto l145 - } - goto l144 - l145: - position, tokenIndex = position144, tokenIndex144 - if !_rules[rulesp]() { - goto l142 - } - if !_rules[ruleclose]() { - goto l142 - } + position++ + if !_rules[rulesp]() { + goto l128 } - l144: - position, tokenIndex = position143, tokenIndex143 - } - { - add(ruleAction31, position) - } - goto l136 - l142: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('f') { - goto l147 - } - position++ - if buffer[position] != rune('a') { - goto l147 - } - position++ - if buffer[position] != rune('l') { - goto l147 - } - position++ - if buffer[position] != rune('s') { - goto l147 - } - position++ - if buffer[position] != rune('e') { - goto l147 - } - position++ - { - position148, tokenIndex148 := position, tokenIndex - { - position149, tokenIndex149 := position, tokenIndex - if !_rules[rulecomma]() { - goto l150 - } - goto l149 - l150: - position, tokenIndex = position149, tokenIndex149 - if !_rules[rulesp]() { - goto l147 - } - if !_rules[ruleclose]() { - goto l147 - } - } - l149: - position, tokenIndex = position148, tokenIndex148 + add(rulelbrack, position132) } { add(ruleAction32, position) } - goto l136 - l147: - position, tokenIndex = position136, tokenIndex136 + if !_rules[rulelist]() { + goto l128 + } { - position153 := position - { - position154, tokenIndex154 := position, tokenIndex - if buffer[position] != rune('-') { - goto l154 - } - position++ - goto l155 - l154: - position, tokenIndex = position154, tokenIndex154 + position134 := position + if !_rules[rulesp]() { + goto l128 } - l155: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 + if buffer[position] != rune(']') { + goto l128 } position++ - l156: - { - position157, tokenIndex157 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l157 - } - position++ - goto l156 - l157: - position, tokenIndex = position157, tokenIndex157 + if !_rules[rulesp]() { + goto l128 } - { - position158, tokenIndex158 := position, tokenIndex - if buffer[position] != rune('.') { - goto l158 - } - position++ - l160: - { - position161, tokenIndex161 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l161 - } - position++ - goto l160 - l161: - position, tokenIndex = position161, tokenIndex161 - } - goto l159 - l158: - position, tokenIndex = position158, tokenIndex158 - } - l159: - add(rulePegText, position153) + add(rulerbrack, position134) } { add(ruleAction33, position) } + } + l130: + add(rulevalue, position129) + } + return true + l128: + position, tokenIndex = position128, tokenIndex128 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position136, tokenIndex136 := position, tokenIndex + { + position137 := position + if !_rules[ruleitem]() { goto l136 - l152: - position, tokenIndex = position136, tokenIndex136 + } + { + position138, tokenIndex138 := position, tokenIndex + if !_rules[rulecomma]() { + goto l138 + } + if !_rules[rulelist]() { + goto l138 + } + goto l139 + l138: + position, tokenIndex = position138, tokenIndex138 + } + l139: + add(rulelist, position137) + } + return true + l136: + position, tokenIndex = position136, tokenIndex136 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action39) / ('"' '"' Action40) / ('\'' '\'' Action41))> */ + func() bool { + position140, tokenIndex140 := position, tokenIndex + { + position141 := position + { + position142, tokenIndex142 := position, tokenIndex + if buffer[position] != rune('n') { + goto l143 + } + position++ + if buffer[position] != rune('u') { + goto l143 + } + position++ + if buffer[position] != rune('l') { + goto l143 + } + position++ + if buffer[position] != rune('l') { + goto l143 + } + position++ { - position164 := position + position144, tokenIndex144 := position, tokenIndex { - position165, tokenIndex165 := position, tokenIndex - if buffer[position] != rune('-') { - goto l165 + position145, tokenIndex145 := position, tokenIndex + if !_rules[rulecomma]() { + goto l146 } - position++ - goto l166 - l165: - position, tokenIndex = position165, tokenIndex165 - } - l166: - if buffer[position] != rune('.') { - goto l163 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 - } - position++ - l167: - { - position168, tokenIndex168 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l168 + goto l145 + l146: + position, tokenIndex = position145, tokenIndex145 + if !_rules[rulesp]() { + goto l143 + } + if !_rules[ruleclose]() { + goto l143 } - position++ - goto l167 - l168: - position, tokenIndex = position168, tokenIndex168 } - add(rulePegText, position164) + l145: + position, tokenIndex = position144, tokenIndex144 } { add(ruleAction34, position) } - goto l136 - l163: - position, tokenIndex = position136, tokenIndex136 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('t') { + goto l148 + } + position++ + if buffer[position] != rune('r') { + goto l148 + } + position++ + if buffer[position] != rune('u') { + goto l148 + } + position++ + if buffer[position] != rune('e') { + goto l148 + } + position++ { - position171 := position + position149, tokenIndex149 := position, tokenIndex { - position174, tokenIndex174 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l175 + position150, tokenIndex150 := position, tokenIndex + if !_rules[rulecomma]() { + goto l151 } - position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l176 + goto l150 + l151: + position, tokenIndex = position150, tokenIndex150 + if !_rules[rulesp]() { + goto l148 } - position++ - goto l174 - l176: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l177 + if !_rules[ruleclose]() { + goto l148 } - position++ - goto l174 - l177: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('-') { - goto l178 - } - position++ - goto l174 - l178: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('_') { - goto l179 - } - position++ - goto l174 - l179: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune(':') { - goto l170 - } - position++ } - l174: - l172: - { - position173, tokenIndex173 := position, tokenIndex - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l173 - } - position++ - } - l180: - goto l172 - l173: - position, tokenIndex = position173, tokenIndex173 - } - add(rulePegText, position171) + l150: + position, tokenIndex = position149, tokenIndex149 } { add(ruleAction35, position) } - goto l136 - l170: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('"') { - goto l187 + goto l142 + l148: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('f') { + goto l153 + } + position++ + if buffer[position] != rune('a') { + goto l153 + } + position++ + if buffer[position] != rune('l') { + goto l153 + } + position++ + if buffer[position] != rune('s') { + goto l153 + } + position++ + if buffer[position] != rune('e') { + goto l153 } position++ { - position188 := position - if !_rules[ruledoublequotedstring]() { - goto l187 + position154, tokenIndex154 := position, tokenIndex + { + position155, tokenIndex155 := position, tokenIndex + if !_rules[rulecomma]() { + goto l156 + } + goto l155 + l156: + position, tokenIndex = position155, tokenIndex155 + if !_rules[rulesp]() { + goto l153 + } + if !_rules[ruleclose]() { + goto l153 + } } - add(rulePegText, position188) + l155: + position, tokenIndex = position154, tokenIndex154 } - if buffer[position] != rune('"') { - goto l187 - } - position++ { add(ruleAction36, position) } - goto l136 - l187: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('\'') { - goto l134 - } - position++ + goto l142 + l153: + position, tokenIndex = position142, tokenIndex142 { - position190 := position - if !_rules[rulesinglequotedstring]() { - goto l134 + position159 := position + { + position160, tokenIndex160 := position, tokenIndex + if buffer[position] != rune('-') { + goto l160 + } + position++ + goto l161 + l160: + position, tokenIndex = position160, tokenIndex160 } - add(rulePegText, position190) + l161: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l158 + } + position++ + l162: + { + position163, tokenIndex163 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 + } + position++ + goto l162 + l163: + position, tokenIndex = position163, tokenIndex163 + } + { + position164, tokenIndex164 := position, tokenIndex + if buffer[position] != rune('.') { + goto l164 + } + position++ + l166: + { + position167, tokenIndex167 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l167 + } + position++ + goto l166 + l167: + position, tokenIndex = position167, tokenIndex167 + } + goto l165 + l164: + position, tokenIndex = position164, tokenIndex164 + } + l165: + add(rulePegText, position159) } - if buffer[position] != rune('\'') { - goto l134 - } - position++ { add(ruleAction37, position) } + goto l142 + l158: + position, tokenIndex = position142, tokenIndex142 + { + position170 := position + { + position171, tokenIndex171 := position, tokenIndex + if buffer[position] != rune('-') { + goto l171 + } + position++ + goto l172 + l171: + position, tokenIndex = position171, tokenIndex171 + } + l172: + if buffer[position] != rune('.') { + goto l169 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l169 + } + position++ + l173: + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l174 + } + position++ + goto l173 + l174: + position, tokenIndex = position174, tokenIndex174 + } + add(rulePegText, position170) + } + { + add(ruleAction38, position) + } + goto l142 + l169: + position, tokenIndex = position142, tokenIndex142 + { + position177 := position + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l176 + } + position++ + } + l180: + l178: + { + position179, tokenIndex179 := position, tokenIndex + { + position186, tokenIndex186 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l187 + } + position++ + goto l186 + l187: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l188 + } + position++ + goto l186 + l188: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l189 + } + position++ + goto l186 + l189: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('-') { + goto l190 + } + position++ + goto l186 + l190: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('_') { + goto l191 + } + position++ + goto l186 + l191: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune(':') { + goto l179 + } + position++ + } + l186: + goto l178 + l179: + position, tokenIndex = position179, tokenIndex179 + } + add(rulePegText, position177) + } + { + add(ruleAction39, position) + } + goto l142 + l176: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + position194 := position + if !_rules[ruledoublequotedstring]() { + goto l193 + } + add(rulePegText, position194) + } + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + add(ruleAction40, position) + } + goto l142 + l193: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + position196 := position + if !_rules[rulesinglequotedstring]() { + goto l140 + } + add(rulePegText, position196) + } + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + add(ruleAction41, position) + } } - l136: - add(ruleitem, position135) + l142: + add(ruleitem, position141) } return true - l134: - position, tokenIndex = position134, tokenIndex134 + l140: + position, tokenIndex = position140, tokenIndex140 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position193 := position - l194: + position199 := position + l200: { - position195, tokenIndex195 := position, tokenIndex + position201, tokenIndex201 := position, tokenIndex { - position196, tokenIndex196 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex { - position198, tokenIndex198 := position, tokenIndex + position204, tokenIndex204 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position205, tokenIndex205 := position, tokenIndex if buffer[position] != rune('"') { - goto l200 + goto l206 } position++ - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 if buffer[position] != rune('\\') { - goto l201 + goto l207 } position++ - goto l199 - l201: - position, tokenIndex = position199, tokenIndex199 + goto l205 + l207: + position, tokenIndex = position205, tokenIndex205 if buffer[position] != rune('\n') { - goto l198 + goto l204 } position++ } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l205: + goto l203 + l204: + position, tokenIndex = position204, tokenIndex204 } if !matchDot() { - goto l197 + goto l203 } - goto l196 - l197: - position, tokenIndex = position196, tokenIndex196 + goto l202 + l203: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l202 + goto l208 } position++ if buffer[position] != rune('n') { - goto l202 + goto l208 } position++ - goto l196 - l202: - position, tokenIndex = position196, tokenIndex196 + goto l202 + l208: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l203 + goto l209 } position++ if buffer[position] != rune('"') { - goto l203 + goto l209 } position++ - goto l196 - l203: - position, tokenIndex = position196, tokenIndex196 + goto l202 + l209: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l204 + goto l210 } position++ if buffer[position] != rune('\'') { - goto l204 + goto l210 } position++ - goto l196 - l204: - position, tokenIndex = position196, tokenIndex196 + goto l202 + l210: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l195 + goto l201 } position++ if buffer[position] != rune('\\') { - goto l195 + goto l201 } position++ } - l196: - goto l194 - l195: - position, tokenIndex = position195, tokenIndex195 + l202: + goto l200 + l201: + position, tokenIndex = position201, tokenIndex201 } - add(ruledoublequotedstring, position193) + add(ruledoublequotedstring, position199) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position206 := position - l207: + position212 := position + l213: { - position208, tokenIndex208 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position209, tokenIndex209 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex { - position211, tokenIndex211 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex { - position212, tokenIndex212 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex if buffer[position] != rune('\'') { - goto l213 + goto l219 } position++ - goto l212 - l213: - position, tokenIndex = position212, tokenIndex212 + goto l218 + l219: + position, tokenIndex = position218, tokenIndex218 if buffer[position] != rune('\\') { - goto l214 + goto l220 } position++ - goto l212 - l214: - position, tokenIndex = position212, tokenIndex212 + goto l218 + l220: + position, tokenIndex = position218, tokenIndex218 if buffer[position] != rune('\n') { - goto l211 + goto l217 } position++ } - l212: - goto l210 - l211: - position, tokenIndex = position211, tokenIndex211 + l218: + goto l216 + l217: + position, tokenIndex = position217, tokenIndex217 } if !matchDot() { - goto l210 + goto l216 } - goto l209 - l210: - position, tokenIndex = position209, tokenIndex209 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l215 + goto l221 } position++ if buffer[position] != rune('n') { - goto l215 + goto l221 } position++ - goto l209 - l215: - position, tokenIndex = position209, tokenIndex209 + goto l215 + l221: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l216 + goto l222 } position++ if buffer[position] != rune('"') { - goto l216 + goto l222 } position++ - goto l209 - l216: - position, tokenIndex = position209, tokenIndex209 + goto l215 + l222: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l217 + goto l223 } position++ if buffer[position] != rune('\'') { - goto l217 + goto l223 } position++ - goto l209 - l217: - position, tokenIndex = position209, tokenIndex209 + goto l215 + l223: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l208 + goto l214 } position++ if buffer[position] != rune('\\') { - goto l208 + goto l214 } position++ } - l209: - goto l207 - l208: - position, tokenIndex = position208, tokenIndex208 + l215: + goto l213 + l214: + position, tokenIndex = position214, tokenIndex214 } - add(rulesinglequotedstring, position206) + add(rulesinglequotedstring, position212) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position218, tokenIndex218 := position, tokenIndex + position224, tokenIndex224 := position, tokenIndex { - position219 := position + position225 := position { - position220, tokenIndex220 := position, tokenIndex + position226, tokenIndex226 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l221 + goto l227 } position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 + goto l226 + l227: + position, tokenIndex = position226, tokenIndex226 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l218 + goto l224 } position++ } - l220: - l222: + l226: + l228: { - position223, tokenIndex223 := position, tokenIndex + position229, tokenIndex229 := position, tokenIndex { - position224, tokenIndex224 := position, tokenIndex + position230, tokenIndex230 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l225 + goto l231 } position++ - goto l224 - l225: - position, tokenIndex = position224, tokenIndex224 + goto l230 + l231: + position, tokenIndex = position230, tokenIndex230 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l226 + goto l232 } position++ - goto l224 - l226: - position, tokenIndex = position224, tokenIndex224 + goto l230 + l232: + position, tokenIndex = position230, tokenIndex230 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l227 - } - position++ - goto l224 - l227: - position, tokenIndex = position224, tokenIndex224 - if buffer[position] != rune('_') { - goto l228 - } - position++ - goto l224 - l228: - position, tokenIndex = position224, tokenIndex224 - if buffer[position] != rune('-') { - goto l223 - } - position++ - } - l224: - goto l222 - l223: - position, tokenIndex = position223, tokenIndex223 - } - add(rulefieldExpr, position219) - } - return true - l218: - position, tokenIndex = position218, tokenIndex218 - return false - }, - /* 17 field <- <(<(fieldExpr / reserved)> Action38)> */ - func() bool { - position229, tokenIndex229 := position, tokenIndex - { - position230 := position - { - position231 := position - { - position232, tokenIndex232 := position, tokenIndex - if !_rules[rulefieldExpr]() { goto l233 } - goto l232 + position++ + goto l230 l233: - position, tokenIndex = position232, tokenIndex232 + position, tokenIndex = position230, tokenIndex230 + if buffer[position] != rune('_') { + goto l234 + } + position++ + goto l230 + l234: + position, tokenIndex = position230, tokenIndex230 + if buffer[position] != rune('-') { + goto l229 + } + position++ + } + l230: + goto l228 + l229: + position, tokenIndex = position229, tokenIndex229 + } + add(rulefieldExpr, position225) + } + return true + l224: + position, tokenIndex = position224, tokenIndex224 + return false + }, + /* 17 field <- <(<(fieldExpr / reserved)> Action42)> */ + func() bool { + position235, tokenIndex235 := position, tokenIndex + { + position236 := position + { + position237 := position + { + position238, tokenIndex238 := position, tokenIndex + if !_rules[rulefieldExpr]() { + goto l239 + } + goto l238 + l239: + position, tokenIndex = position238, tokenIndex238 { - position234 := position + position240 := position { - position235, tokenIndex235 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex if buffer[position] != rune('_') { - goto l236 + goto l242 } position++ if buffer[position] != rune('r') { - goto l236 + goto l242 } position++ if buffer[position] != rune('o') { - goto l236 + goto l242 } position++ if buffer[position] != rune('w') { - goto l236 + goto l242 } position++ - goto l235 - l236: - position, tokenIndex = position235, tokenIndex235 + goto l241 + l242: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l237 + goto l243 } position++ if buffer[position] != rune('c') { - goto l237 + goto l243 } position++ if buffer[position] != rune('o') { - goto l237 + goto l243 } position++ if buffer[position] != rune('l') { - goto l237 + goto l243 } position++ - goto l235 - l237: - position, tokenIndex = position235, tokenIndex235 + goto l241 + l243: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l238 + goto l244 } position++ if buffer[position] != rune('s') { - goto l238 + goto l244 } position++ if buffer[position] != rune('t') { - goto l238 + goto l244 } position++ if buffer[position] != rune('a') { - goto l238 + goto l244 } position++ if buffer[position] != rune('r') { - goto l238 + goto l244 } position++ if buffer[position] != rune('t') { - goto l238 + goto l244 } position++ - goto l235 - l238: - position, tokenIndex = position235, tokenIndex235 + goto l241 + l244: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l239 + goto l245 } position++ if buffer[position] != rune('e') { - goto l239 + goto l245 } position++ if buffer[position] != rune('n') { - goto l239 + goto l245 } position++ if buffer[position] != rune('d') { - goto l239 + goto l245 } position++ - goto l235 - l239: - position, tokenIndex = position235, tokenIndex235 + goto l241 + l245: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l240 + goto l246 } position++ if buffer[position] != rune('t') { - goto l240 + goto l246 } position++ if buffer[position] != rune('i') { - goto l240 + goto l246 } position++ if buffer[position] != rune('m') { - goto l240 + goto l246 } position++ if buffer[position] != rune('e') { - goto l240 + goto l246 } position++ if buffer[position] != rune('s') { - goto l240 + goto l246 } position++ if buffer[position] != rune('t') { - goto l240 + goto l246 } position++ if buffer[position] != rune('a') { - goto l240 + goto l246 } position++ if buffer[position] != rune('m') { - goto l240 + goto l246 } position++ if buffer[position] != rune('p') { - goto l240 + goto l246 } position++ - goto l235 - l240: - position, tokenIndex = position235, tokenIndex235 + goto l241 + l246: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l229 + goto l235 } position++ if buffer[position] != rune('f') { - goto l229 + goto l235 } position++ if buffer[position] != rune('i') { - goto l229 + goto l235 } position++ if buffer[position] != rune('e') { - goto l229 + goto l235 } position++ if buffer[position] != rune('l') { - goto l229 + goto l235 } position++ if buffer[position] != rune('d') { - goto l229 + goto l235 } position++ } - l235: - add(rulereserved, position234) + l241: + add(rulereserved, position240) } } - l232: - add(rulePegText, position231) + l238: + add(rulePegText, position237) } { - add(ruleAction38, position) + add(ruleAction42, position) } - add(rulefield, position230) + add(rulefield, position236) } return true - l229: - position, tokenIndex = position229, tokenIndex229 + l235: + position, tokenIndex = position235, tokenIndex235 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action39)> */ + /* 19 posfield <- <( Action43)> */ func() bool { - position243, tokenIndex243 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position244 := position + position250 := position { - position245 := position + position251 := position if !_rules[rulefieldExpr]() { - goto l243 + goto l249 } - add(rulePegText, position245) + add(rulePegText, position251) } { - add(ruleAction39, position) + add(ruleAction43, position) } - add(ruleposfield, position244) + add(ruleposfield, position250) } return true - l243: - position, tokenIndex = position243, tokenIndex243 + l249: + position, tokenIndex = position249, tokenIndex249 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position247, tokenIndex247 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position248 := position + position254 := position { - position249, tokenIndex249 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l250 + goto l256 } position++ - l251: + l257: { - position252, tokenIndex252 := position, tokenIndex + position258, tokenIndex258 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l252 + goto l258 } position++ - goto l251 - l252: - position, tokenIndex = position252, tokenIndex252 + goto l257 + l258: + position, tokenIndex = position258, tokenIndex258 } - goto l249 - l250: - position, tokenIndex = position249, tokenIndex249 + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 if buffer[position] != rune('0') { - goto l247 + goto l253 } position++ } - l249: - add(ruleuint, position248) + l255: + add(ruleuint, position254) } return true - l247: - position, tokenIndex = position247, tokenIndex247 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, - /* 21 uintrow <- <( Action40)> */ + /* 21 uintrow <- <( Action44)> */ nil, - /* 22 col <- <(( Action41) / ('\'' '\'' Action42) / ('"' '"' Action43))> */ + /* 22 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ func() bool { - position254, tokenIndex254 := position, tokenIndex + position260, tokenIndex260 := position, tokenIndex { - position255 := position + position261 := position { - position256, tokenIndex256 := position, tokenIndex + position262, tokenIndex262 := position, tokenIndex { - position258 := position + position264 := position if !_rules[ruleuint]() { - goto l257 + goto l263 } - add(rulePegText, position258) + add(rulePegText, position264) } { - add(ruleAction41, position) + add(ruleAction45, position) } - goto l256 - l257: - position, tokenIndex = position256, tokenIndex256 + goto l262 + l263: + position, tokenIndex = position262, tokenIndex262 if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + position267 := position + if !_rules[rulesinglequotedstring]() { + goto l266 + } + add(rulePegText, position267) + } + if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + add(ruleAction46, position) + } + goto l262 + l266: + position, tokenIndex = position262, tokenIndex262 + if buffer[position] != rune('"') { goto l260 } position++ { - position261 := position - if !_rules[rulesinglequotedstring]() { + position269 := position + if !_rules[ruledoublequotedstring]() { goto l260 } - add(rulePegText, position261) + add(rulePegText, position269) } - if buffer[position] != rune('\'') { + if buffer[position] != rune('"') { goto l260 } position++ { - add(ruleAction42, position) - } - goto l256 - l260: - position, tokenIndex = position256, tokenIndex256 - if buffer[position] != rune('"') { - goto l254 - } - position++ - { - position263 := position - if !_rules[ruledoublequotedstring]() { - goto l254 - } - add(rulePegText, position263) - } - if buffer[position] != rune('"') { - goto l254 - } - position++ - { - add(ruleAction43, position) + add(ruleAction47, position) } } - l256: - add(rulecol, position255) + l262: + add(rulecol, position261) } return true - l254: - position, tokenIndex = position254, tokenIndex254 + l260: + position, tokenIndex = position260, tokenIndex260 return false }, - /* 23 row <- <(( Action44) / ('\'' '\'' Action45) / ('"' '"' Action46))> */ + /* 23 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ nil, /* 24 open <- <('(' sp)> */ func() bool { - position266, tokenIndex266 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex { - position267 := position + position273 := position if buffer[position] != rune('(') { - goto l266 + goto l272 } position++ if !_rules[rulesp]() { - goto l266 + goto l272 } - add(ruleopen, position267) + add(ruleopen, position273) } return true - l266: - position, tokenIndex = position266, tokenIndex266 + l272: + position, tokenIndex = position272, tokenIndex272 return false }, /* 25 close <- <(')' sp)> */ func() bool { - position268, tokenIndex268 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position269 := position + position275 := position if buffer[position] != rune(')') { - goto l268 + goto l274 } position++ if !_rules[rulesp]() { - goto l268 + goto l274 } - add(ruleclose, position269) + add(ruleclose, position275) } return true - l268: - position, tokenIndex = position268, tokenIndex268 + l274: + position, tokenIndex = position274, tokenIndex274 return false }, /* 26 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position271 := position - l272: + position277 := position + l278: { - position273, tokenIndex273 := position, tokenIndex + position279, tokenIndex279 := position, tokenIndex { - position274, tokenIndex274 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex if buffer[position] != rune(' ') { - goto l275 + goto l281 } position++ - goto l274 - l275: - position, tokenIndex = position274, tokenIndex274 + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 if buffer[position] != rune('\t') { - goto l276 + goto l282 } position++ - goto l274 - l276: - position, tokenIndex = position274, tokenIndex274 + goto l280 + l282: + position, tokenIndex = position280, tokenIndex280 if buffer[position] != rune('\n') { - goto l273 + goto l279 } position++ } - l274: - goto l272 - l273: - position, tokenIndex = position273, tokenIndex273 + l280: + goto l278 + l279: + position, tokenIndex = position279, tokenIndex279 } - add(rulesp, position271) + add(rulesp, position277) } return true }, /* 27 comma <- <(sp ',' sp)> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position278 := position + position284 := position if !_rules[rulesp]() { - goto l277 + goto l283 } if buffer[position] != rune(',') { - goto l277 + goto l283 } position++ if !_rules[rulesp]() { - goto l277 + goto l283 } - add(rulecomma, position278) + add(rulecomma, position284) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, /* 28 lbrack <- <('[' sp)> */ @@ -2682,139 +2792,139 @@ func (p *PQL) Init() { nil, /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position282, tokenIndex282 := position, tokenIndex + position288, tokenIndex288 := position, tokenIndex { - position283 := position + position289 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if buffer[position] != rune('-') { - goto l282 + goto l288 } position++ { - position284, tokenIndex284 := position, tokenIndex + position290, tokenIndex290 := position, tokenIndex if buffer[position] != rune('0') { - goto l285 + goto l291 } position++ - goto l284 - l285: - position, tokenIndex = position284, tokenIndex284 + goto l290 + l291: + position, tokenIndex = position290, tokenIndex290 if buffer[position] != rune('1') { - goto l282 + goto l288 } position++ } - l284: + l290: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if buffer[position] != rune('-') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if buffer[position] != rune('T') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if buffer[position] != rune(':') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l288 } position++ - add(ruletimestampbasicfmt, position283) + add(ruletimestampbasicfmt, position289) } return true - l282: - position, tokenIndex = position282, tokenIndex282 + l288: + position, tokenIndex = position288, tokenIndex288 return false }, /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position286, tokenIndex286 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex { - position287 := position + position293 := position { - position288, tokenIndex288 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex if buffer[position] != rune('"') { - goto l289 + goto l295 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l289 + goto l295 } if buffer[position] != rune('"') { - goto l289 + goto l295 } position++ - goto l288 - l289: - position, tokenIndex = position288, tokenIndex288 + goto l294 + l295: + position, tokenIndex = position294, tokenIndex294 if buffer[position] != rune('\'') { - goto l290 + goto l296 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l290 + goto l296 } if buffer[position] != rune('\'') { - goto l290 + goto l296 } position++ - goto l288 - l290: - position, tokenIndex = position288, tokenIndex288 + goto l294 + l296: + position, tokenIndex = position294, tokenIndex294 if !_rules[ruletimestampbasicfmt]() { - goto l286 + goto l292 } } - l288: - add(ruletimestampfmt, position287) + l294: + add(ruletimestampfmt, position293) } return true - l286: - position, tokenIndex = position286, tokenIndex286 + l292: + position, tokenIndex = position292, tokenIndex292 return false }, - /* 33 timestamp <- <( Action47)> */ + /* 33 timestamp <- <( Action51)> */ nil, /* 35 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2832,86 +2942,94 @@ func (p *PQL) Init() { nil, /* 42 Action7 <- <{p.endCall()}> */ nil, - /* 43 Action8 <- <{p.startCall("TopN")}> */ + /* 43 Action8 <- <{p.startCall("ClearRow")}> */ nil, /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 45 Action10 <- <{p.startCall("Range")}> */ + /* 45 Action10 <- <{p.startCall("Store")}> */ nil, /* 46 Action11 <- <{p.endCall()}> */ nil, + /* 47 Action12 <- <{p.startCall("TopN")}> */ nil, - /* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action13 <- <{p.endCall()}> */ nil, - /* 49 Action13 <- <{ p.endCall() }> */ + /* 49 Action14 <- <{p.startCall("Range")}> */ nil, - /* 50 Action14 <- <{ p.addBTWN() }> */ + /* 50 Action15 <- <{p.endCall()}> */ nil, - /* 51 Action15 <- <{ p.addLTE() }> */ nil, - /* 52 Action16 <- <{ p.addGTE() }> */ + /* 52 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 53 Action17 <- <{ p.addEQ() }> */ + /* 53 Action17 <- <{ p.endCall() }> */ nil, - /* 54 Action18 <- <{ p.addNEQ() }> */ + /* 54 Action18 <- <{ p.addBTWN() }> */ nil, - /* 55 Action19 <- <{ p.addLT() }> */ + /* 55 Action19 <- <{ p.addLTE() }> */ nil, - /* 56 Action20 <- <{ p.addGT() }> */ + /* 56 Action20 <- <{ p.addGTE() }> */ nil, - /* 57 Action21 <- <{p.startConditional()}> */ + /* 57 Action21 <- <{ p.addEQ() }> */ nil, - /* 58 Action22 <- <{p.endConditional()}> */ + /* 58 Action22 <- <{ p.addNEQ() }> */ nil, - /* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action23 <- <{ p.addLT() }> */ nil, - /* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action24 <- <{ p.addGT() }> */ nil, - /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action25 <- <{p.startConditional()}> */ nil, - /* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action26 <- <{p.endConditional()}> */ nil, - /* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action28 <- <{ p.startList() }> */ + /* 64 Action28 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 65 Action29 <- <{ p.endList() }> */ + /* 65 Action29 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 66 Action30 <- <{ p.addVal(nil) }> */ + /* 66 Action30 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 67 Action31 <- <{ p.addVal(true) }> */ + /* 67 Action31 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 68 Action32 <- <{ p.addVal(false) }> */ + /* 68 Action32 <- <{ p.startList() }> */ nil, - /* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action33 <- <{ p.endList() }> */ nil, - /* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action34 <- <{ p.addVal(nil) }> */ nil, - /* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addVal(true) }> */ nil, - /* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addVal(false) }> */ nil, - /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 75 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 76 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action42 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 79 Action43 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action43 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, /* 80 Action44 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 81 Action45 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 81 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 82 Action46 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 82 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action47 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 83 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 84 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 85 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 86 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 87 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/roaring/containers.go b/roaring/containers.go index 5275d4f04..ed745a915 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -51,13 +51,13 @@ func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n if i < 0 { c := NewContainer() c.containerType = containerType - c.n = n + c.n = int32(n) c.mapped = mapped sc.insertAt(key, c, -i-1) } else { c := sc.containers[i] c.containerType = containerType - c.n = n + c.n = int32(n) c.mapped = mapped } diff --git a/roaring/roaring.go b/roaring/roaring.go index c3f2cc3db..ed7be6be1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -46,7 +46,7 @@ const ( // at the beginning of every serialized run container. runCountHeaderSize = 2 - // interval32Size is the size of a single run in a container.runs. + // interval16Size is the size of a single run in a container.runs. interval16Size = 4 // bitmapN is the number of values in a container.bitmap. @@ -73,6 +73,7 @@ type Containers interface { // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. + // TODO(2.0) make n int32 PutContainerValues(key uint64, containerType byte, n int, mapped bool) // Remove takes the container at key out. @@ -242,7 +243,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { if found && skey == ekey { citer.Next() _, c := citer.Value() - return uint64(c.countRange(int(lowbits(start)), int(lowbits(end)))) + return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end)))) } for citer.Next() { @@ -252,7 +253,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { panic("should be impossible for k to be less than skey") } if k == skey { - n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1)) + n += uint64(c.countRange(int32(lowbits(start)), maxContainerVal+1)) continue } if k < ekey { @@ -260,7 +261,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { continue } if k == ekey { - n += uint64(c.countRange(0, int(lowbits(end)))) + n += uint64(c.countRange(0, int32(lowbits(end)))) break } if k > ekey { @@ -805,7 +806,7 @@ type Iterator struct { citer ContainerIterator key uint64 c *Container - j, k int // i: container; j: array index, bit index, or run index; k: offset within the run + j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run } // Seek moves to the first value equal to or greater than `seek`. @@ -830,7 +831,7 @@ func (itr *Iterator) Seek(seek uint64) { if itr.j < 0 { itr.j = -itr.j - 1 } - if itr.j < len(itr.c.array) { + if itr.j < int32(len(itr.c.array)) { itr.j-- return } @@ -853,7 +854,7 @@ func (itr *Iterator) Seek(seek uint64) { j, contains := binSearchRuns(lb, itr.c.runs) if contains { itr.j = j - itr.k = int(lb) - int(itr.c.runs[j].start) - 1 + itr.k = int32(lb) - int32(itr.c.runs[j].start) - 1 } else { // Set iterator to next value in the Bitmap. itr.j = j @@ -864,7 +865,7 @@ func (itr *Iterator) Seek(seek uint64) { } // If it's a bitmap container then move to index before the value and call next(). - itr.j = int(lb) - 1 + itr.j = int32(lb) - 1 } // Next returns the next value in the bitmap. @@ -914,14 +915,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } r := itr.c.runs[itr.j] - runLength := int(r.last - r.start) + runLength := int32(r.last - r.start) if itr.k >= runLength { // Reached end of run, move to the next run. itr.j, itr.k = itr.j+1, -1 } - if itr.j >= len(itr.c.runs) { + if itr.j >= int32(len(itr.c.runs)) { // Reached end of runs, move to the next container. if !itr.citer.Next() { itr.c = nil @@ -942,7 +943,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Find first non-zero bit in current bitmap, if possible. hb := itr.j >> 6 - if hb >= len(itr.c.bitmap) { + if hb >= int32(len(itr.c.bitmap)) { if !itr.citer.Next() { itr.c = nil return 0, true @@ -953,14 +954,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64) if lb != 0 { - itr.j = itr.j + trailingZeroN(lb) + itr.j = itr.j + int32(trailingZeroN(lb)) return itr.peek(), false } // Otherwise iterate through remaining bitmaps to find next bit. - for hb++; hb < len(itr.c.bitmap); hb++ { + for hb++; hb < int32(len(itr.c.bitmap)); hb++ { if itr.c.bitmap[hb] != 0 { - itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb]) + itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap[hb])) return itr.peek(), false } } @@ -1008,7 +1009,7 @@ const runMaxSize = 2048 type Container struct { mapped bool // mapped directly to a byte slice when true containerType byte // array, bitmap, or run - n int // number of integers in container + n int32 // number of integers in container array []uint16 // used for array containers bitmap []uint64 // used for bitmap containers runs []interval16 // used for RLE containers @@ -1020,8 +1021,8 @@ type interval16 struct { } // runlen returns the count of integers in the interval. -func (iv interval16) runlen() int { - return 1 + int(iv.last-iv.start) +func (iv interval16) runlen() int32 { + return 1 + int32(iv.last-iv.start) } // newContainer returns a new instance of container. @@ -1035,12 +1036,12 @@ func (c *Container) Mapped() bool { } // N returns the cached bit count of the container -func (c *Container) N() int { +func (c *Container) N() int32 { return c.n } // Update updates the container -func (c *Container) Update(containerType byte, n int, mapped bool) { +func (c *Container) Update(containerType byte, n int32, mapped bool) { c.containerType = containerType c.n = n c.mapped = mapped @@ -1088,12 +1089,12 @@ func (c *Container) unmap() { } // count counts all bits in the container. -func (c *Container) count() (n int) { +func (c *Container) count() (n int32) { return c.countRange(0, maxContainerVal+1) } // countRange counts the number of bits set between [start, end). -func (c *Container) countRange(start, end int) (n int) { +func (c *Container) countRange(start, end int32) (n int32) { if c.isArray() { return c.arrayCountRange(start, end) } else if c.isRun() { @@ -1102,10 +1103,10 @@ func (c *Container) countRange(start, end int) (n int) { return c.bitmapCountRange(start, end) } -func (c *Container) arrayCountRange(start, end int) (n int) { - i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start }) - for ; i < len(c.array); i++ { - v := int(c.array[i]) +func (c *Container) arrayCountRange(start, end int32) (n int32) { + i := int32(sort.Search(len(c.array), func(i int) bool { return int32(c.array[i]) >= start })) + for ; i < int32(len(c.array)); i++ { + v := int32(c.array[i]) if v >= end { break } @@ -1114,14 +1115,14 @@ func (c *Container) arrayCountRange(start, end int) (n int) { return n } -func (c *Container) bitmapCountRange(start, end int) int { +func (c *Container) bitmapCountRange(start, end int32) int32 { var n uint64 i, j := start/64, end/64 // Special case when start and end fall in the same word. if i == j { offi, offj := uint(start%64), uint(64-end%64) n += popcount((c.bitmap[i] >> offi) << (offj + offi)) - return int(n) + return int32(n) } // Count partial starting word. @@ -1136,39 +1137,39 @@ func (c *Container) bitmapCountRange(start, end int) int { } // Count partial ending word. - if j < len(c.bitmap) { + if j < int32(len(c.bitmap)) { off := 64 - (uint(end) % 64) n += popcount(c.bitmap[j] << off) } - return int(n) + return int32(n) } -func (c *Container) runCountRange(start, end int) (n int) { +func (c *Container) runCountRange(start, end int32) (n int32) { for _, iv := range c.runs { // iv is before range - if int(iv.last) < start { + if int32(iv.last) < start { continue } // iv is after range - if end < int(iv.start) { + if end < int32(iv.start) { break } // iv is superset of range - if int(iv.start) < start && int(iv.last) > end { + if int32(iv.start) < start && int32(iv.last) > end { return end - start } // iv is subset of range - if int(iv.start) >= start && int(iv.last) < end { + if int32(iv.start) >= start && int32(iv.last) < end { n += iv.runlen() } // iv overlaps beginning of range - if int(iv.start) < start && int(iv.last) < end { - n += int(iv.last) - start + 1 + if int32(iv.start) < start && int32(iv.last) < end { + n += int32(iv.last) - start + 1 } // iv overlaps end of range - if int(iv.start) > start && int(iv.last) >= end { - n += end - int(iv.start) + if int32(iv.start) > start && int32(iv.last) >= end { + n += end - int32(iv.start) } } return n @@ -1286,34 +1287,34 @@ func (c *Container) Contains(v uint16) bool { } } -func (c *Container) bitmapCountRuns() (r int) { +func (c *Container) bitmapCountRuns() (r int32) { for i := 0; i < 1023; i++ { v, v1 := c.bitmap[i], c.bitmap[i+1] - r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1)) + r = r + int32(popcount((v<<1)&^v)+((v>>63)&^v1)) } vl := c.bitmap[len(c.bitmap)-1] - r = r + int(popcount((vl<<1)&^vl)+vl>>63) + r = r + int32(popcount((vl<<1)&^vl)+vl>>63) return r } -func (c *Container) arrayCountRuns() (r int) { - prev := -2 +func (c *Container) arrayCountRuns() (r int32) { + prev := int32(-2) for _, v := range c.array { - if prev+1 != int(v) { + if prev+1 != int32(v) { r++ } - prev = int(v) + prev = int32(v) } return r } -func (c *Container) countRuns() (r int) { +func (c *Container) countRuns() (r int32) { if c.isArray() { return c.arrayCountRuns() } else if c.isBitmap() { return c.bitmapCountRuns() } else if c.isRun() { - return len(c.runs) + return int32(len(c.runs)) } // sure hope this never happens @@ -1369,10 +1370,10 @@ func (c *Container) bitmapContains(v uint16) bool { // binSearchRuns returns the index of the run containing v, and true, when v is contained; // or the index of the next run starting after v, and false, when v is not contained. -func binSearchRuns(v uint16, a []interval16) (int, bool) { - i := sort.Search(len(a), - func(i int) bool { return a[i].last >= v }) - if i < len(a) { +func binSearchRuns(v uint16, a []interval16) (int32, bool) { + i := int32(sort.Search(len(a), + func(i int) bool { return a[i].last >= v })) + if i < int32(len(a)) { return i, (v >= a[i].start) && (v <= a[i].last) } @@ -1698,7 +1699,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) //} - // Write sizeof(uint32) * cardinality bytes. + // Write sizeof(uint16) * cardinality bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) return int64(nn), err } @@ -1767,7 +1768,7 @@ func (c *Container) check() error { var a ErrorList if c.isArray() { - if len(c.array) != c.n { + if int32(len(c.array)) != c.n { a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n)) } } else if c.isRun() { @@ -1796,7 +1797,7 @@ func (c *Container) check() error { type containerInfo struct { Key uint64 // container key Type string // container type (array, bitmap, or run) - N int // number of bits + N int32 // number of bits Alloc int // memory used Pointer unsafe.Pointer // offset within the mmap } @@ -1838,7 +1839,7 @@ func flipRun(b *Container) *Container { return flipBitmap(x) } -func intersectionCount(a, b *Container) int { +func intersectionCount(a, b *Container) int32 { if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) @@ -1866,7 +1867,7 @@ func intersectionCount(a, b *Container) int { } } -func intersectionCountArrayArray(a, b *Container) (n int) { +func intersectionCountArrayArray(a, b *Container) (n int32) { na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1882,7 +1883,7 @@ func intersectionCountArrayArray(a, b *Container) (n int) { return n } -func intersectionCountArrayRun(a, b *Container) (n int) { +func intersectionCountArrayRun(a, b *Container) (n int32) { na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -1898,7 +1899,7 @@ func intersectionCountArrayRun(a, b *Container) (n int) { return n } -func intersectionCountRunRun(a, b *Container) (n int) { +func intersectionCountRunRun(a, b *Container) (n int32) { na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -1910,33 +1911,33 @@ func intersectionCountRunRun(a, b *Container) (n int) { j++ } else if va.last > vb.last && va.start >= vb.start { // |--vb-|-|-va--| - n += 1 + int(vb.last-va.start) + n += 1 + int32(vb.last-va.start) j++ } else if va.last > vb.last && va.start < vb.start { // |--va|--vb--|--| - n += 1 + int(vb.last-vb.start) + n += 1 + int32(vb.last-vb.start) j++ } else if va.last <= vb.last && va.start >= vb.start { // |--vb|--va--|--| - n += 1 + int(va.last-va.start) + n += 1 + int32(va.last-va.start) i++ } else if va.last <= vb.last && va.start < vb.start { // |--va-|-|-vb--| - n += 1 + int(va.last-vb.start) + n += 1 + int32(va.last-vb.start) i++ } } return n } -func intersectionCountBitmapRun(a, b *Container) (n int) { +func intersectionCountBitmapRun(a, b *Container) (n int32) { for _, iv := range b.runs { - n += a.bitmapCountRange(int(iv.start), int(iv.last)+1) + n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1) } return n } -func intersectionCountArrayBitmap(a, b *Container) (n int) { +func intersectionCountArrayBitmap(a, b *Container) (n int32) { ln := len(b.bitmap) for _, val := range a.array { i := int(val >> 6) @@ -1944,13 +1945,13 @@ func intersectionCountArrayBitmap(a, b *Container) (n int) { break } off := val % 64 - n += int(b.bitmap[i]>>off) & 1 + n += int32(b.bitmap[i]>>off) & 1 } return n } -func intersectionCountBitmapBitmap(a, b *Container) (n int) { - return int(popcountAndSlice(a.bitmap, b.bitmap)) +func intersectionCountBitmapBitmap(a, b *Container) (n int32) { + return int32(popcountAndSlice(a.bitmap, b.bitmap)) } func intersect(a, b *Container) *Container { @@ -1995,7 +1996,7 @@ func intersectArrayArray(a, b *Container) *Container { i, j = i+1, j+1 } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2016,7 +2017,7 @@ func intersectArrayRun(a, b *Container) *Container { i++ } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2050,7 +2051,7 @@ func intersectRunRun(a, b *Container) *Container { i++ } } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -2076,7 +2077,7 @@ func intersectBitmapRun(a, b *Container) *Container { } } } - output.n = len(output.array) + output.n = int32(len(output.array)) } else { // right now this iterates through the runs and sets integers in the // bitmap that are in the runs. alternately, we could zero out ranges in @@ -2093,22 +2094,22 @@ func intersectBitmapRun(a, b *Container) *Container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b output.bitmap[i] = a.bitmap[i] - output.n += int(popcount(a.bitmap[i])) + output.n += int32(popcount(a.bitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := a.bitmap[i] & mask output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (a.bitmap[i] >> offset) << offset output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (a.bitmap[i] << offset) >> offset output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } // update loop vars i++ @@ -2134,20 +2135,30 @@ func intersectArrayBitmap(a, b *Container) *Container { output.array = append(output.array, va) } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } func intersectBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} - - for i := range a.bitmap { - v := a.bitmap[i] & b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) - + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + n int32 + ) + for i := 0; i < bitmapN; i++ { + ob[i] = ab[i] & bb[i] + n += int32(popcount(ob[i])) + } + + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, } - output.optimize() return output } @@ -2249,10 +2260,10 @@ func unionArrayRun(a, b *Container) *Container { // interval is earlier than the start of the last interval in the list of runs. // Its return value is the amount by which the cardinality of the container was // increased. -func (c *Container) runAppendInterval(v interval16) int { +func (c *Container) runAppendInterval(v interval16) int32 { if len(c.runs) == 0 { c.runs = append(c.runs, v) - return int(v.last-v.start) + 1 + return int32(v.last-v.start) + 1 } last := c.runs[len(c.runs)-1] @@ -2261,10 +2272,10 @@ func (c *Container) runAppendInterval(v interval16) int { } if last.last+1 >= v.start && v.last > last.last { c.runs[len(c.runs)-1].last = v.last - return int(v.last - last.last) + return int32(v.last - last.last) } else if last.last+1 < v.start { c.runs = append(c.runs, v) - return int(v.last-v.start) + 1 + return int32(v.last-v.start) + 1 } return 0 } @@ -2328,16 +2339,16 @@ func (c *Container) bitmapSetRange(i, j uint64) { xcnt := popcount(X) ycnt := popcount(Y) if x == y { - c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y))) + c.n += int32((j - i) - popcount(c.bitmap[x]&(X&Y))) c.bitmap[x] |= (X & Y) } else { - c.n += int(xcnt - popcount(c.bitmap[x]&X)) + c.n += int32(xcnt - popcount(c.bitmap[x]&X)) c.bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int(64 - popcount(c.bitmap[i])) + c.n += int32(64 - popcount(c.bitmap[i])) c.bitmap[i] = maxBitmap } - c.n += int(ycnt - popcount(c.bitmap[y]&Y)) + c.n += int32(ycnt - popcount(c.bitmap[y]&Y)) c.bitmap[y] |= Y } } @@ -2351,19 +2362,19 @@ func (c *Container) bitmapXorRange(i, j uint64) { if x == y { cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip - c.n += int(popcount(c.bitmap[x]) - cnt) + c.n += int32(popcount(c.bitmap[x]) - cnt) } else { cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= X - c.n += int(popcount(c.bitmap[x]) - cnt) + c.n += int32(popcount(c.bitmap[x]) - cnt) for i := x + 1; i < y; i++ { cnt = popcount(c.bitmap[i]) c.bitmap[i] ^= maxBitmap - c.n += int(popcount(c.bitmap[i]) - cnt) + c.n += int32(popcount(c.bitmap[i]) - cnt) } cnt = popcount(c.bitmap[y]) c.bitmap[y] ^= Y - c.n += int(popcount(c.bitmap[y]) - cnt) + c.n += int32(popcount(c.bitmap[y]) - cnt) } } @@ -2374,16 +2385,16 @@ func (c *Container) bitmapZeroRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - c.n -= int(popcount(c.bitmap[x] & (X & Y))) + c.n -= int32(popcount(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) } else { - c.n -= int(popcount(c.bitmap[x] & X)) + c.n -= int32(popcount(c.bitmap[x] & X)) c.bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int(popcount(c.bitmap[i])) + c.n -= int32(popcount(c.bitmap[i])) c.bitmap[i] = 0 } - c.n -= int(popcount(c.bitmap[y] & Y)) + c.n -= int32(popcount(c.bitmap[y] & Y)) c.bitmap[y] &= ^Y } } @@ -2437,17 +2448,28 @@ func unionArrayBitmap(a, b *Container) *Container { } func unionBitmapBitmap(a, b *Container) *Container { - output := &Container{ - bitmap: make([]uint64, bitmapN), - containerType: containerBitmap, - } + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + + n int32 + ) for i := 0; i < bitmapN; i++ { - v := a.bitmap[i] | b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) + ob[i] = ab[i] | bb[i] + n += int32(popcount(ob[i])) } + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, + } return output } @@ -2519,7 +2541,7 @@ func differenceArrayRun(a, b *Container) *Container { j := 0 // run index // handle overlap - for i < a.n { + for i < int(a.n) { // keep all array elements before beginning of runs if a.array[i] < b.runs[j].start { @@ -2551,7 +2573,7 @@ func differenceArrayRun(a, b *Container) *Container { output.array = append(output.array, a.array[i:]...) // TODO: consider handling container.n mutations in one place // like we do with container.add(). - output.n += len(a.array[i:]) + output.n += int32(len(a.array[i:])) } else { for _, v := range a.array[i:] { output.add(v) @@ -2609,7 +2631,7 @@ RUNLOOP: continue } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) - output.n += int(vb - start) + output.n += int32(vb - start) if vb == 65535 { // overflow break RUNLOOP } @@ -2623,7 +2645,7 @@ RUNLOOP: if start <= run.last { output.runs = append(output.runs, interval16{start: start, last: run.last}) - output.n += int(run.last - start + 1) + output.n += int32(run.last - start + 1) } } output.optimize() @@ -2679,7 +2701,7 @@ func differenceRunBitmap(a, b *Container) *Container { } } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -2763,7 +2785,7 @@ func differenceArrayBitmap(a, b *Container) *Container { output.array = append(output.array, va) } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2783,13 +2805,27 @@ func differenceBitmapArray(a, b *Container) *Container { } func differenceBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html - for i := range a.bitmap { - v := a.bitmap[i] & (^b.bitmap[i]) - output.bitmap[i] = v - output.n += int(popcount(v)) + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + n int32 + ) + + for i := 0; i < bitmapN; i++ { + ob[i] = ab[i] & (^bb[i]) + n += int32(popcount(ob[i])) + } + + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, } if output.n < ArrayMaxSize { output.bitmapToArray() @@ -2874,16 +2910,28 @@ func xorArrayBitmap(a, b *Container) *Container { } func xorBitmapBitmap(a, b *Container) *Container { - output := &Container{ - bitmap: make([]uint64, bitmapN), - containerType: containerBitmap, - } + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + + n int32 + ) + for i := 0; i < bitmapN; i++ { - v := a.bitmap[i] ^ b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) + ob[i] = ab[i] ^ bb[i] + n += int32(popcount(ob[i])) } + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, + } if output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -2962,9 +3010,9 @@ func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } // search32 returns the index of value in a. If value is not found, it works the // same way as search64. -func search32(a []uint16, value uint16) int { +func search32(a []uint16, value uint16) int32 { // Optimize for elements and the last element. - n := len(a) + n := int32(len(a)) if n == 0 { return -1 } else if a[n-1] == value { @@ -2972,9 +3020,9 @@ func search32(a []uint16, value uint16) int { } // Otherwise perform binary search for exact match. - lo, hi := 0, n-1 + lo, hi := int32(0), n-1 for lo+16 <= hi { - i := int(uint((lo + hi)) >> 1) + i := int32(uint((lo + hi)) >> 1) v := a[i] if v < value { @@ -3280,7 +3328,7 @@ func xorRunRun(a, b *Container) *Container { } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -3295,7 +3343,7 @@ func xorBitmapRun(a, b *Container) *Container { output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -3336,9 +3384,14 @@ func popcount(x uint64) uint64 { } func popcountAndSlice(s, m []uint64) uint64 { + var ( + a = s[:bitmapN] + b = m[:bitmapN] + ) + cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] & m[i]) + for i := 0; i < bitmapN; i++ { + cnt += popcount(a[i] & b[i]) } return cnt } @@ -3494,7 +3547,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { c.runs = nil c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n] - pos += c.n * 2 + pos += int(c.n * 2) case containerBitmap: c.array = nil c.runs = nil diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index b049d40cf..a4c162629 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -39,7 +39,7 @@ func TestRunAppendInterval(t *testing.T) { tests := []struct { base []interval16 app interval16 - exp int + exp int32 }{ { base: []interval16{}, @@ -207,10 +207,10 @@ func TestRunContains(t *testing.T) { func TestBitmapCountRange(t *testing.T) { c := Container{containerType: containerBitmap} tests := []struct { - start int - end int + start int32 + end int32 bitmap []uint64 - exp int + exp int32 }{ {start: 0, end: 1, bitmap: []uint64{1}, exp: 1}, {start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2}, @@ -251,7 +251,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) { b.bitmapToRun() res = intersectRunRun(a, b) n := intersectionCountRunRun(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) { + if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) { t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) } } @@ -261,7 +261,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { tests := []struct { array []uint16 bitmap []uint64 - exp int + exp int32 }{ { array: []uint16{0}, @@ -384,7 +384,7 @@ func TestIntersectionCountRunRun(t *testing.T) { tests := []struct { aruns []interval16 bruns []interval16 - exp int + exp int32 }{ { aruns: []interval16{}, @@ -478,7 +478,7 @@ func TestIntersectRunRun(t *testing.T) { aruns []interval16 bruns []interval16 exp []interval16 - expN int + expN int32 }{ { aruns: []interval16{}, @@ -540,7 +540,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{1}, @@ -602,7 +602,7 @@ func TestIntersectBitmapRunArray(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint16 - expN int + expN int32 }{ { bitmap: []uint64{1}, @@ -761,11 +761,11 @@ func TestDifferenceMixed(t *testing.T) { a.containerType = containerRun b.array = []uint16{0, 2, 4, 6, 8, 10, 12} - b.n = len(b.array) + b.n = int32(len(b.array)) b.containerType = containerArray d.array = []uint16{1, 3, 5, 7, 9, 11, 12} - d.n = len(d.array) + d.n = int32(len(d.array)) d.containerType = containerArray res := difference(a, b) @@ -784,11 +784,11 @@ func TestDifferenceMixed(t *testing.T) { t.Fatalf("test #3 expected empty but got %v", res.runs) } - c.bitmap = []uint64{0x64} + c.bitmap = MakeBitmap([]uint64{0x64}) c.n = c.countRange(0, 100) c.containerType = containerBitmap res = difference(c, a) - if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) { + if !reflect.DeepEqual(res.bitmap, MakeBitmap([]uint64{0x4})) { t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) } @@ -939,7 +939,7 @@ func TestBitmapSetRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000FFF900}, @@ -991,7 +991,7 @@ func TestArrayToBitmap(t *testing.T) { copy(exp, test.exp) a.array = test.array - a.n = len(test.array) + a.n = int32(len(test.array)) a.arrayToBitmap() if !reflect.DeepEqual(a.bitmap, exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) @@ -1016,10 +1016,10 @@ func TestBitmapToArray(t *testing.T) { } for i, test := range tests { a.bitmap = make([]uint64, bitmapN) - n := 0 + n := int32(0) for i, v := range test.bitmap { a.bitmap[i] = v - n += int(popcount(v)) + n += int32(popcount(v)) } a.n = n @@ -1067,7 +1067,7 @@ func TestRunToBitmap(t *testing.T) { } a.runs = test.runs - a.n = n + a.n = int32(n) a.runToBitmap() if !reflect.DeepEqual(a.bitmap, exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) @@ -1149,7 +1149,7 @@ func TestBitmapToRun(t *testing.T) { a.bitmap[i] = v n += int(popcount(v)) } - a.n = n + a.n = int32(n) x := a.bitmap a.bitmapToRun() if !reflect.DeepEqual(a.runs, test.exp) { @@ -1188,7 +1188,7 @@ func TestArrayToRun(t *testing.T) { for i, test := range tests { a.array = test.array - a.n = int(len(test.array)) + a.n = int32(len(test.array)) a.arrayToRun() if !reflect.DeepEqual(a.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) @@ -1222,7 +1222,7 @@ func TestRunToArray(t *testing.T) { for i, test := range tests { a.runs = test.runs - a.n = len(test.exp) + a.n = int32(len(test.exp)) a.runToArray() if !reflect.DeepEqual(a.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) @@ -1237,7 +1237,7 @@ func TestBitmapZeroRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000FFFF00}, @@ -1279,7 +1279,7 @@ func TestUnionBitmapRun(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{2}, @@ -1313,7 +1313,7 @@ func TestBitmapCountRuns(t *testing.T) { c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 - exp int + exp int32 }{ { bitmap: []uint64{0xFF00FF00}, @@ -1361,7 +1361,7 @@ func TestArrayCountRuns(t *testing.T) { c := &Container{containerType: containerArray} tests := []struct { array []uint16 - exp int + exp int32 }{ { array: []uint16{}, @@ -1414,7 +1414,7 @@ func TestDifferenceArrayRun(t *testing.T) { } for i, test := range tests { a.array = test.array - a.n = len(a.array) + a.n = int32(len(a.array)) b.runs = test.runs b.n = b.runCountRange(0, 100) ret := differenceArrayRun(a, b) @@ -1482,7 +1482,7 @@ func TestDifferenceRunArray(t *testing.T) { a.runs = test.runs a.n = a.runCountRange(0, 100) b.array = test.array - b.n = len(b.array) + b.n = int32(len(b.array)) ret := differenceRunArray(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -1730,7 +1730,7 @@ func TestDifferenceRunRun(t *testing.T) { aruns []interval16 bruns []interval16 exp []interval16 - expn int + expn int32 }{ { // this tests all six overlap combinations @@ -2009,7 +2009,7 @@ func TestBitmapXorRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000000000}, @@ -2302,7 +2302,7 @@ func TestRunBinSearchContains(t *testing.T) { runs []interval16 index uint16 exp struct { - index int + index int32 found bool } }{ @@ -2310,7 +2310,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}}, index: uint16(3), exp: struct { - index int + index int32 found bool }{index: 0, found: true}, }, @@ -2318,7 +2318,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}}, index: uint16(13), exp: struct { - index int + index int32 found bool }{index: 0, found: false}, }, @@ -2326,7 +2326,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, index: uint16(13), exp: struct { - index int + index int32 found bool }{index: 0, found: false}, }, @@ -2334,7 +2334,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, index: uint16(36), exp: struct { - index int + index int32 found bool }{index: 1, found: false}, }, @@ -2355,7 +2355,7 @@ func TestRunBinSearch(t *testing.T) { runs []interval16 search uint16 exp bool - expi int + expi int32 }{ { runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, diff --git a/server.go b/server.go index cde1dd406..3bb3ce1b2 100644 --- a/server.go +++ b/server.go @@ -234,6 +234,13 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } +func OptServerTranslateFileMapSize(mapSize int) ServerOption { + return func(s *Server) error { + s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ @@ -330,20 +337,20 @@ func (s *Server) Open() error { // Initialize id-key storage. if err := s.holder.translateFile.Open(); err != nil { - return err + return errors.Wrap(err, "opening TranslateFile") } // Open Cluster management. if err := s.cluster.waitForStarted(); err != nil { - return fmt.Errorf("opening Cluster: %v", err) + return errors.Wrap(err, "opening Cluster") } // Open holder. if err := s.holder.Open(); err != nil { - return fmt.Errorf("opening Holder: %v", err) + return errors.Wrap(err, "opening Holder") } if err := s.cluster.setNodeState(nodeStateReady); err != nil { - return fmt.Errorf("setting nodeState: %v", err) + return errors.Wrap(err, "setting nodeState") } // Listen for joining nodes. @@ -474,7 +481,9 @@ func (s *Server) receiveMessage(m Message) error { if f == nil { return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field) } - f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)) + if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { + return errors.Wrap(err, "adding remote available shards") + } case *CreateIndexMessage: opt := obj.Meta _, err := s.holder.CreateIndex(obj.Index, *opt) @@ -633,13 +642,15 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { for _, fs := range is.Fields { f := s.holder.Field(is.Name, fs.Name) - // if we don't know about an field locally, log a error because + // if we don't know about a field locally, log an error because // fields should be created and synced prior to shard creation if f == nil { s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name) continue } - f.addRemoteAvailableShards(fs.AvailableShards) + if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } } } @@ -664,6 +675,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.nodeID) s.diagnostics.Set("ClusterID", s.cluster.id) + s.diagnostics.EnrichWithCPUInfo() s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval diff --git a/server/config.go b/server/config.go index 8367eb835..d255e4bb6 100644 --- a/server/config.go +++ b/server/config.go @@ -71,8 +71,9 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` - // DEPRECATED: Translation config supports translation store replication. Translation struct { + MapSize int `toml:"map-size"` + // DEPRECATED: Translation config supports translation store replication. PrimaryURL string `toml:"primary-url"` } `toml:"translation"` diff --git a/server/handler_test.go b/server/handler_test.go index 2c1aa9868..9e76d77b9 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) { }) + t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) { + // Roaring import into a non-set field should fail. + if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData)) + req.Header.Set("Content-Type", "application/x-binary") + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + }) + t.Run("Status", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) diff --git a/server/server.go b/server/server.go index 4fdd711fe..b4ac595fa 100644 --- a/server/server.go +++ b/server/server.go @@ -283,6 +283,14 @@ func (m *Command) SetupServer() error { coordinatorOpt, } + if m.Config.Translation.MapSize > 0 { + serverOptions = append( + serverOptions, + pilosa.OptServerTranslateFileMapSize( + m.Config.Translation.MapSize, + ), + ) + } serverOptions = append(serverOptions, m.serverOptions...) m.Server, err = pilosa.NewServer(serverOptions...) diff --git a/server/server_test.go b/server/server_test.go index ab860dc1b..aa202a54c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "math/rand" "reflect" "sort" @@ -403,6 +404,7 @@ func TestClusteringNodesReplica1(t *testing.T) { // Create new main with the same config. config := cluster[2].Command.Config + config.Translation.MapSize = 100000 // config.Bind = cluster[2].API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster @@ -476,6 +478,7 @@ func TestClusteringNodesReplica2(t *testing.T) { // Create new main with the same config. config := cluster[2].Command.Config + config.Translation.MapSize = 100000 // config.Bind = cluster[2].API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster @@ -496,6 +499,7 @@ func TestClusteringNodesReplica2(t *testing.T) { // Create new main with the same config. config = cluster[1].Command.Config // config.Bind = cluster[1].API.Node().URI.HostPort() + config.Translation.MapSize = 100000 // this isn't necessary, but makes the test run way faster config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port)) @@ -564,4 +568,128 @@ func TestRemoveNodeAfterItDies(t *testing.T) { } } +// Ensure program imports timestamps as UTC. +func TestMain_ImportTimestamp(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + indexName := "i" + fieldName := "f" + + // Create index. + if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + // Create field. + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil { + t.Fatal(err) + } + + data := pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + RowIDs: []uint64{1, 2}, + ColumnIDs: []uint64{1, 2}, + Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00 + } + + // Import data. + if err := m.API.Import(context.Background(), &data); err != nil { + t.Fatal(err) + } + + // Ensure the correct views were created. + dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName) + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + exp := []string{ + "standard", "standard_2018", "standard_201801", "standard_20180101", + "standard_2019", "standard_201912", "standard_20191231", + } + got := []string{} + for _, f := range files { + got = append(got, f.Name()) + } + + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected %v, but got %v", exp, got) + } +} + +func TestClusterQueriesAfterRestart(t *testing.T) { + cluster := test.MustRunCluster(t, 3) + defer cluster.Close() + cmd1 := cluster[1] + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + // build a query to set the first bit in 100 shards + query := strings.Builder{} + for i := 0; i < 100; i++ { + query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth)) + } + _, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: query.String(), + }) + if err != nil { + t.Fatalf("setting 100 bits in 100 shards: %v", err) + } + + results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: "Count(Row(testfield=0))", + }) + if err != nil { + t.Fatalf("counting row: %v", err) + } + if results.Results[0].(uint64) != 100 { + t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0]) + } + + err = cmd1.Command.Close() + if err != nil { + t.Fatalf("closing node0: %v", err) + } + + // confirm that cluster stops accepting queries after one node closes + if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { + t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) + } + + // Create new main with the same config. + config := cmd1.Command.Config + config.Bind = cmd1.API.Node().URI.HostPort() + + // this isn't necessary, but makes the test run way faster + config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port)) + cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr) + cmd1.Command.Config = config + err = cmd1.Start() + if err != nil { + t.Fatalf("reopening node 0: %v", err) + } + + for cmd1.API.State() != pilosa.ClusterStateNormal { + time.Sleep(time.Millisecond) + } + + results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: "Count(Row(testfield=0))", + }) + if err != nil { + t.Fatalf("counting row: %v", err) + } + if results.Results[0].(uint64) != 100 { + t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0]) + } +} + // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address. diff --git a/test/pilosa.go b/test/pilosa.go index 9dbd78057..27331d53f 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -55,16 +55,21 @@ func newCommand(opts ...server.CommandOption) *Command { panic(err) } - // set aggressive close timeout by default to avoid hanging tests. This was + // Set aggressive close timeout by default to avoid hanging tests. This was // a problem with PDK tests which used go-pilosa as well. We put it at the // beginning of the option slice so that it can be overridden by user-passed // options. - opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...) + // Also set TranslateFile MapSize to a smaller number so memory allocation + // does not fail on 32-bit systems. + opts = append([]server.CommandOption{ + server.OptCommandCloseTimeout(time.Millisecond * 2), + }, opts...) m := &Command{commandOptions: opts} m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true + m.Config.Translation.MapSize = 100000 if testing.Verbose() { m.Command.Stdout = os.Stdout diff --git a/translate.go b/translate.go index 41f3b4852..cb16de929 100644 --- a/translate.go +++ b/translate.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "io" "io/ioutil" @@ -17,6 +16,7 @@ import ( "time" "github.com/cespare/xxhash" + "github.com/pkg/errors" ) const ( @@ -81,9 +81,29 @@ type TranslateFile struct { replicationRetryInterval time.Duration } +// TranslateFileOption is a functional option type for pilosa.TranslateFile +type TranslateFileOption func(f *TranslateFile) error + +func OptTranslateFileMapSize(mapSize int) TranslateFileOption { + return func(f *TranslateFile) error { + f.mapSize = mapSize + return nil + } +} + // NewTranslateFile returns a new instance of TranslateFile. -func NewTranslateFile() *TranslateFile { - return &TranslateFile{ +func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { + var defaultMapSize64 int64 = 10 * (1 << 30) + var defaultMapSize int + + if ^uint(0)>>32 > 0 { + // 10GB default map size + defaultMapSize = int(defaultMapSize64) + } else { + // Use 2GB default map size on 32-bit systems + defaultMapSize = (1 << 31) - 1 + } + f := &TranslateFile{ writeNotify: make(chan struct{}), closing: make(chan struct{}), cols: make(map[string]*index), @@ -96,25 +116,35 @@ func NewTranslateFile() *TranslateFile { replicationRetryInterval: defaultReplicationRetryInterval, } + + for _, opt := range opts { + err := opt(f) + if err != nil { + // TODO (2.0): Change func signature to return error + panic(errors.Wrap(err, "applying option")) + } + } + + return f } func (s *TranslateFile) Open() (err error) { // Open writer & buffered writer. if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { - return err + return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { - return err + return errors.Wrapf(err, "open file %s", s.Path) } s.w = bufio.NewWriter(s.file) // Memory map data file. if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { - return err + return errors.Wrapf(err, "creating Mmap (size: %d)", s.mapSize) } // Replay the log. if err := s.replayEntries(); err != nil { - return err + return errors.Wrap(err, "replaying log entries") } // Listen to primaryStoreEvents channel. diff --git a/translate_mapsize_386.go b/translate_mapsize_386.go deleted file mode 100644 index 259b735ac..000000000 --- a/translate_mapsize_386.go +++ /dev/null @@ -1,5 +0,0 @@ -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so must be < 2^31 -const defaultMapSize = (1 << 31) - 1 // 2GB diff --git a/translate_mapsize_all64bitsystems.go b/translate_mapsize_all64bitsystems.go deleted file mode 100644 index 605d6270a..000000000 --- a/translate_mapsize_all64bitsystems.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !386 - -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on -// 64bit systems. -const defaultMapSize = 10 * (1 << 30) // 10GB diff --git a/translate_test.go b/translate_test.go index 706fe365f..1aea26cfe 100644 --- a/translate_test.go +++ b/translate_test.go @@ -367,7 +367,7 @@ func TestPrintTranslateFile(t *testing.T) { } f.Close() - s := pilosa.NewTranslateFile() + s := pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25)) s.Path = f.Name() err = s.Open() if err != nil { @@ -809,7 +809,7 @@ func NewTranslateFile() *TranslateFile { } f.Close() - s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))} s.Path = f.Name() return s } @@ -847,7 +847,7 @@ func (s *TranslateFile) Reopen() error { } s.lock.Lock() - s.TranslateFile = pilosa.NewTranslateFile() + s.TranslateFile = pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25)) s.lock.Unlock() s.Path = prev.Path s.SetPrimaryStore("restored-primary", prev.PrimaryTranslateStore) diff --git a/view.go b/view.go index 2feac8b44..5fc0f5eb8 100644 --- a/view.go +++ b/view.go @@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment { frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) + } else if v.fieldType == FieldTypeBool { + frag.mutexVector = newBoolVector(frag) } return frag }