Merge branch 'master' into new-rows-iterate

This commit is contained in:
Matt Jaffee 2018-10-09 12:27:05 -05:00
commit 36a539d24e
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
45 changed files with 3572 additions and 1638 deletions

View file

@ -68,9 +68,20 @@ jobs:
docker:
- image: circleci/python:2.7-jessie
steps:
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
- *fast-checkout
- run: sudo pip install awscli
- run: make prerelease-upload
dockerhub-upload:
<<: *defaults
steps:
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
- *fast-checkout
- setup_remote_docker
- run: make docker
- run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
- run: docker login -u $DOCKER_USER -p $DOCKER_PASS
- run: docker push pilosa/pilosa:master
workflows:
version: 2
test:
@ -104,3 +115,7 @@ workflows:
- prerelease-upload:
requires:
- prerelease
- dockerhub-upload:
requires:
- linter
- test-golang-1.10

82
api.go
View file

@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
return QueryResponse{}, errors.Wrap(err, "validating api method")
}
resp := QueryResponse{}
q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
if err != nil {
return resp, errors.Wrap(err, "parsing")
return QueryResponse{}, errors.Wrap(err, "parsing")
}
execOpts := &execOptions{
Remote: req.Remote,
@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
}
results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {
return resp, errors.Wrap(err, "executing")
return QueryResponse{}, errors.Wrap(err, "executing")
}
resp.Results = results
// Fill column attributes if requested.
// execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true)
if execOpts.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Row)
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs)
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if api.holder.translateFile != nil {
for _, col := range resp.ColumnAttrSets {
v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
return resp, nil
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
ax := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if len(attrs) == 0 {
continue
}
// Append column with attributes.
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return ax, nil
}
// CreateIndex makes a new Pilosa index.
func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) {
if err := api.validate(apiCreateIndex); err != nil {
@ -321,13 +263,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
nodes := api.cluster.shardNodes(indexName, shard)
var eg errgroup.Group
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound)
}
// only set fields are supported
if field.Type() != FieldTypeSet {
return NewBadRequestError(errors.New("roaring import is only supported for set fields"))
}
for _, node := range nodes {
node := node
if node.ID == api.server.nodeID {
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound)
}
// must make a copy of data to operate on locally. field.importRoaring changes data
d2 := make([]byte, len(data))
copy(d2, data)
@ -732,7 +680,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error {
if ts == 0 {
continue
}
t := time.Unix(0, ts)
t := time.Unix(0, ts).UTC()
timestamps[i] = &t
}
@ -801,6 +749,8 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error {
}
// MaxShards returns the maximum shard number for each index in a map.
// TODO (2.0): This method has been deprecated. Instead, use
// AvailableShardsByIndex.
func (api *API) MaxShards(_ context.Context) map[string]uint64 {
m := make(map[string]uint64)
for k, v := range api.holder.availableShardsByIndex() {

View file

@ -167,8 +167,10 @@ func NewRankCache(maxEntries uint32) *rankCache {
func (c *rankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Ignore if the column count is below the threshold.
if n < c.thresholdValue {
// Ignore if the column count is below the threshold,
// unless the count is 0, which is effectively used
// to clear the cache value.
if n < c.thresholdValue && n > 0 {
return
}

View file

@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
tests := []commandTest{
// TEST 0
{
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"},
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"},
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
@ -65,13 +65,14 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
v.Check(cmd.Server.Config.Translation.MapSize, 100000)
return v.Error()
},
},
// TEST 1
{
args: []string{"server", "--anti-entropy.interval", "9m0s"},
env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110"},
env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"},
cfgFileContent: `
bind = "localhost:0"
data-dir = "` + actualDataDir + `"
@ -85,12 +86,13 @@ func TestServerConfig(t *testing.T) {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
v.Check(cmd.Server.Config.Translation.MapSize, 100000)
return v.Error()
},
},
// TEST 2
{
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:19444"

View file

@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
}
// Ensure that import into bool field runs.
func TestImportCommand_RunBool(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
ctx := context.Background()
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.API.Node().URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`)))
cm.Index = "i"
cm.Field = "f"
t.Run("Valid", func(t *testing.T) {
file, err := ioutil.TempFile("", "import-bool.csv")
if err != nil {
t.Fatal(err)
}
file.Write([]byte("0,1\n1,2\n1,3"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run to bool field doesn't work: %s", err)
}
})
// Ensure that invalid bool values return an error.
t.Run("Invalid", func(t *testing.T) {
file, err := ioutil.TempFile("", "import-invalid-bool.csv")
if err != nil {
t.Fatal(err)
}
file.Write([]byte("0,1\n1,2\n1,3\n2,4"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") {
t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err)
}
})
}

View file

@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
// Translation
flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")

View file

@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool {
return false
}
// EnrichWithCPUInfo adds CPU information to the diagnostics payload.
func (d *diagnosticsCollector) EnrichWithCPUInfo() {
d.Set("CPUArch", d.server.systemInfo.CPUArch())
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *diagnosticsCollector) EnrichWithOSInfo() {
uptime, err := d.server.systemInfo.Uptime()
@ -265,6 +270,7 @@ type SystemInfo interface {
MemFree() (uint64, error)
MemTotal() (uint64, error)
MemUsed() (uint64, error)
CPUArch() string
}
// newNopSystemInfo creates a no-op implementation of SystemInfo.
@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) {
func (n *nopSystemInfo) MemUsed() (uint64, error) {
return 0, nil
}
// CPUArch returns the CPU architecture, such as amd64
func (n *nopSystemInfo) CPUArch() string {
return ""
}

View file

@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the
pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
```
##### Importing Boolean Values
If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true).
For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row.
```
0,3
0,9
1,1
1,2
1,4
1,8
```
<div class="note">
<p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
</div>

View file

@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option
* `int`
* `min` (int): Minimum integer value allowed for the field.
* `max` (int): Maximum integer value allowed for the field.
* `bool`
* (boolean fields take no arguments)
* `time`
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
* `mutex`

View file

@ -307,6 +307,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
skip-verify = true
```
#### Translation Map Size
* Description: Size in bytes of mmap to allocate for key translation
* Flag: `translation.map-size`
* Env: `PILOSA_TRANSLATION_MAP_SIZE`
* Config:
```toml
[translation]
map-size = 10737418240
```
### Example Cluster Configuration
A three node cluster running on different hosts could be minimally configured as follows:

View file

@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust
### Field Type
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`.
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`.
#### Set
@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00)
#### Mutex
Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field.
#### Boolean
A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values.

View file

@ -22,7 +22,7 @@ nav = []
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.

View file

@ -236,6 +236,70 @@ Clear(10, stargazer=1)
This represents removing the relationship between the user with id=1 and the repository with id=10.
#### ClearRow
**Spec:**
```
ClearRow(<FIELD>=<ROW>)
```
**Description:**
`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns.
**Result Type:** boolean
A return value of `true` indicates that at least one column was toggled from 1 to 0.
A return value of `false` indicates that all bits in the row were already 0 and nothing changed.
**Examples:**
Clear all bit in row 1 in the stargazer field:
```request
ClearRow(stargazer=1)
```
```response
{"results":[true]}
```
This represents removing the relationship between the user with id=1 and all repositories.
#### Store
**Spec:**
```
Store(<ROW_CALL>, <FIELD>=<ROW>)
```
**Description:**
`Store` writes the results of <ROW_CALL> to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
**Result Type:** boolean
Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call.
**Examples:**
Store the contents of stargazer row 1 into stargazer row 2:
```request
Store(Row(stargazer=1), stargazer=2)
```
```response
{"results":[true]}
```
Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20.
```request
Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20)
```
```response
{"results":[true]}
```
### Read Operations
#### Row

View file

@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \
{"success":true}
```
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
``` request

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package proto
import (
@ -812,30 +826,32 @@ func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent)
func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
m.Node = &pilosa.Node{}
decodeIndexStatuses(pb.Indexes, m.Indexes)
m.Indexes = decodeIndexStatuses(pb.Indexes)
m.Schema = &pilosa.Schema{}
decodeSchema(pb.Schema, m.Schema)
}
func decodeIndexStatuses(a []*internal.IndexStatus, m []*pilosa.IndexStatus) {
m = m[:0]
func decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.IndexStatus {
m := make([]*pilosa.IndexStatus, 0)
for i := range a {
m = append(m, &pilosa.IndexStatus{})
decodeIndexStatus(a[i], m[i])
}
return m
}
func decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) {
m.Name = pb.Name
decodeFieldStatuses(pb.Fields, m.Fields)
m.Fields = decodeFieldStatuses(pb.Fields)
}
func decodeFieldStatuses(a []*internal.FieldStatus, m []*pilosa.FieldStatus) {
m = m[:0]
func decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.FieldStatus {
m := make([]*pilosa.FieldStatus, 0)
for i := range a {
m = append(m, &pilosa.FieldStatus{})
decodeFieldStatus(a[i], m[i])
}
return m
}
func decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) {

View file

@ -88,13 +88,13 @@ func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Containe
// this struct is added to prevent the closure locals from being escaped out to the heap
type updater struct {
key uint64
n int
n int32
containerType byte
mapped bool
}
func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
a := updater{key, n, containerType, mapped}
a := updater{key, int32(n), containerType, mapped}
btc.tree.Put(key, a.update)
}

View file

@ -81,20 +81,21 @@ func newExecutor(opts ...executorOption) *executor {
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
resp := QueryResponse{}
// Verify that an index is set.
if index == "" {
return nil, ErrIndexRequired
return resp, ErrIndexRequired
}
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
return resp, ErrIndexNotFound
}
// Verify that the number of writes do not exceed the maximum.
if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest {
return nil, ErrTooManyWrites
return resp, ErrTooManyWrites
}
// Default options.
@ -107,14 +108,48 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
if !opt.Remote {
for i := range q.Calls {
if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
return nil, err
return resp, err
}
}
}
results, err := e.execute(ctx, index, q, shards, opt)
if err != nil {
return nil, err
return resp, err
}
resp.Results = results
// Fill column attributes if requested.
if opt.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Row)
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs)
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if idx.Keys() {
for _, col := range columnAttrSets {
v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
// Translate response objects from ids to keys, if necessary.
@ -123,11 +158,35 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
for i := range results {
results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
if err != nil {
return nil, err
return resp, err
}
}
}
return results, nil
return resp, nil
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
ax := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if len(attrs) == 0 {
continue
}
// Append column with attributes.
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return ax, nil
}
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
@ -184,6 +243,10 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeMax(ctx, index, c, shards, opt)
case "Clear":
return e.executeClearBit(ctx, index, c, opt)
case "ClearRow":
return e.executeClearRow(ctx, index, c, shards, opt)
case "Store":
return e.executeSetRow(ctx, index, c, shards, opt)
case "Count":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, shards, opt)
@ -1055,8 +1118,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca
rowID, rowOK, rowErr := c.UintArg(fieldName)
if rowErr != nil {
return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr)
}
if !rowOK {
} else if !rowOK {
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
@ -1429,7 +1491,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
}
// executeClearBitField executes a Clear() call for a single view.
// executeClearBitField executes a Clear() call for a field.
func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) {
shard := colID / ShardWidth
ret := false
@ -1459,9 +1521,170 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
return ret, nil
}
// executeClearRow executes a ClearRow() call.
func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
// Ensure the field type supports ClearRow().
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("ClearRow() argument required: field")
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
switch field.Type() {
case FieldTypeSet, FieldTypeTime, FieldTypeMutex, FieldTypeBool:
// These field types support ClearRow().
default:
return false, fmt.Errorf("ClearRow() is not supported on %s field types", field.Type())
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeClearRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
val := v.(bool)
if prev == nil {
return val
}
return val || prev.(bool)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
return result.(bool), err
}
// executeClearRowShard executes a ClearRow() call for a single shard.
func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("ClearRow() argument required: field")
}
// Read fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading ClearRow() row: %v", err)
} else if !ok {
return false, fmt.Errorf("ClearRow() row argument '%v' required", rowLabel)
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
// Remove the row from all views.
changed := false
for _, view := range field.views() {
fragment := e.Holder.fragment(index, fieldName, view.name, shard)
if fragment == nil {
continue
}
cleared, err := fragment.clearRow(rowID)
if err != nil {
return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard)
}
changed = changed || cleared
}
return changed, nil
}
// executeSetRow executes a SetRow() call.
func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
// Ensure the field type supports SetRow().
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("SetRow() argument required: field")
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
if field.Type() != FieldTypeSet {
return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type())
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeSetRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
val := v.(bool)
if prev == nil {
return val
}
return val || prev.(bool)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
return result.(bool), err
}
// executeSetRowShard executes a SetRow() call for a single shard.
func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("SetRow() argument required: field")
}
// Read fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading SetRow() row: %v", err)
} else if !ok {
return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel)
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return false, ErrFieldNotFound
}
// Retrieve source row.
var src *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return false, errors.Wrap(err, "getting source row")
}
src = row
} else {
return false, errors.New("SetRow() requires a source row")
}
// Set the row on the standard view.
changed := false
fragment := e.Holder.fragment(index, fieldName, viewStandard, shard)
if fragment == nil {
// Since the destination fragment doesn't exist, create one.
view, err := field.createViewIfNotExists(viewStandard)
if err != nil {
return false, errors.Wrap(err, "creating view")
}
fragment, err = view.createFragmentIfNotExists(shard)
if err != nil {
return false, errors.Wrapf(err, "creating fragment: %d", shard)
}
}
set, err := fragment.setRow(src, rowID)
if err != nil {
return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard)
}
changed = changed || set
return changed, nil
}
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
// Read colID.
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
@ -1493,8 +1716,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
}
}
// Int field.
if f.Type() == FieldTypeInt {
// Read remaining fields using labels.
// Read row value.
rowVal, ok, err := c.IntArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
@ -1503,27 +1727,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
}
return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
} else {
// Read remaining fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// Read row ID.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// executeSetBitField executes a Set() call for a specific field.
@ -1954,21 +2178,22 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
var colKey, rowKey, fieldName string
if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" {
switch c.Name {
case "Set", "Clear", "Row", "Range", "SetColumnAttrs":
// Positional args in new PQL syntax require special handling here.
colKey = "_" + columnLabel
fieldName, _ = c.FieldArg()
rowKey = fieldName
} else if c.Name == "SetRowAttrs" {
case "SetRowAttrs":
// Positional args in new PQL syntax require special handling here.
rowKey = "_" + rowLabel
fieldName = callArgString(c, "_field")
} else if c.Name == "Rows" {
case "Rows":
fieldName = callArgString(c, "field")
rowKey = "previous"
} else if c.Name == "GroupBy" {
case "GroupBy":
return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy")
} else {
default:
colKey = "col"
fieldName = callArgString(c, "field")
rowKey = "row"
@ -2001,7 +2226,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
// will raise an error downstream when it's used.
return nil
}
if field.keys() {
// Bool field keys do not use the translator because there
// are only two possible values. Instead, they are handled
// directly.
if field.Type() == FieldTypeBool {
boolVal, err := callArgBool(c, rowKey)
if err != nil {
return errors.Wrap(err, "getting bool key")
}
rowID := falseRowID
if boolVal {
rowID = trueRowID
}
c.Args[rowKey] = rowID
} else if field.keys() {
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
return errors.New("row value must be a string when field 'keys' option enabled")
}
@ -2268,6 +2507,18 @@ func (vc *ValCount) larger(other ValCount) ValCount {
}
}
func callArgBool(call *pql.Call, key string) (bool, error) {
value, ok := call.Args[key]
if !ok {
return false, errors.New("missing bool argument")
}
b, ok := value.(bool)
if !ok {
return false, fmt.Errorf("invalid bool argument type: %T", value)
}
return b, nil
}
func callArgString(call *pql.Call, key string) string {
value, ok := call.Args[key]
if !ok {

View file

@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
})
}
// Ensure a set query can be executed on a bool field.
func TestExecutor_Execute_SetBool(t *testing.T) {
t.Run("Basic", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
t.Fatal(err)
}
// Set a true bit.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
t.Fatal(err)
} else if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
// Set the same bit to true again verify nothing changed.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
t.Fatal(err)
} else if res.Results[0].(bool) {
t.Fatalf("expected column to be unchanged")
}
// Set the same bit to false.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil {
t.Fatal(err)
} else if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
// Ensure that the false row is set.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil {
t.Fatal(err)
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) {
t.Fatalf("unexpected colums: %+v", columns)
}
// Ensure that the true row is empty.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil {
t.Fatal(err)
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected colums: %+v", columns)
}
})
t.Run("Error", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
t.Fatal(err)
}
// Set bool using a string value.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil {
t.Fatalf("expected invalid bool type error")
}
// Set bool using an integer.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil {
t.Fatalf("expected invalid bool type error")
}
})
}
// Ensure old PQL syntax doesn't break anything too badly.
func TestExecutor_Execute_OldPQL(t *testing.T) {
c := test.MustRunCluster(t, 1)
@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create felds.
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil {
t.Fatal(err)
@ -948,6 +1020,58 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
}
// Ensure a range query with keys can be executed.
func TestExecutor_Execute_Range_WithKeys(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
// Set columns.
cc := `
Set(2, f="foo", 1999-12-31T00:00)
Set(3, f="foo", 2000-01-01T00:00)
Set(4, f="foo", 2000-01-02T00:00)
Set(5, f="foo", 2000-02-01T00:00)
Set(6, f="foo", 2001-01-01T00:00)
Set(7, f="foo", 2002-01-01T02:00)
Set(2, f="foo", 1999-12-30T00:00)
Set(2, f="foo", 2002-02-01T00:00)
Set(2, f="bar", 2001-01-01T00:00)
`
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f="foo")`}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
}
// Ensure a Range(bsiGroup) query can be executed.
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
c := test.MustRunCluster(t, 1)
@ -1453,6 +1577,36 @@ func TestExecutor_QueryCall(t *testing.T) {
}
})
t.Run("columnAttrsWithKeys", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Set columns for rows 0, 10, & 20 across two shards.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set("one-hundred", f="ten")
SetColumnAttrs("one-hundred", foo="bar")
`}); err != nil {
t.Fatal(err)
}
targetColAttrSets := []*pilosa.ColumnAttrSet{
{Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}},
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f="ten"), columnAttrs=true)`}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) {
t.Fatalf("unexpected keys: %+v", keys)
} else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
t.Run("shards", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -1601,6 +1755,401 @@ func TestExecutor_Execute_Not(t *testing.T) {
}
}
// Ensure a row can be cleared.
func TestExecutor_Execute_ClearRow(t *testing.T) {
t.Run("Set", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Clear the row and ensure we get a `true` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Clear the row again and ensure we get a `false` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Ensure the row is empty.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Ensure other rows were not affected.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Mutex", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0))
if err != nil {
t.Fatal(err)
}
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Clear the row and ensure we get a `true` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Clear the row again and ensure we get a `false` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Ensure the row is empty.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Ensure other rows were not affected.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Time", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD")))
if err != nil {
t.Fatal(err)
}
// Set columns.
cc := `
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)
`
rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`
rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
// Clear the row and ensure we get a `true` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Ensure the row is empty.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
}
// Ensure other rows were not affected.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil {
t.Fatal(err)
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Int", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100))
if err != nil {
t.Fatal(err)
}
// Ensure that clearing a row raises an error.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil {
t.Fatal("expected clear row to return an error")
}
})
t.Run("TopN", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
cc := `
Set(2, f=1)
Set(3, f=1)
Set(4, f=1)
Set(5, f=1)
Set(6, f=1)
Set(7, f=1)
Set(8, f=1)
Set(2, f=2)
Set(3, f=2)
Set(4, f=2)
Set(5, f=2)
Set(6, f=2)
Set(7, f=2)
Set(2, f=3)
Set(3, f=3)
Set(4, f=3)
Set(5, f=3)
Set(6, f=3)
`
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
t.Fatal(err)
}
if err := c[0].RecalculateCaches(); err != nil {
t.Fatalf("recalculating caches: %v", err)
}
// Check the TopN results.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
{ID: 1, Count: 7},
{ID: 2, Count: 6},
{ID: 3, Count: 5},
}}) {
t.Fatalf("topn wrong results: %v", res.Results)
}
// Clear the row and ensure we get a `true` response.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache).
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
{ID: 1, Count: 7},
{ID: 3, Count: 5},
}}) {
t.Fatalf("topn wrong results: %v", res.Results)
}
})
}
// Ensure a row can be set.
func TestExecutor_Execute_SetRow(t *testing.T) {
t.Run("Set_NewRow", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := index.CreateField("tmp", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10),
}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Store row 10 into a different row.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected set row result: %+v", res)
}
// Ensure the row was populated.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Set_NoSource", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10),
}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Store row 9 (which doesn't exist) into a different row.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected set row result: %+v", res)
}
// Ensure the row was populated.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Store row 9 (which doesn't exist) into a row that does exist.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected set row result: %+v", res)
}
// Ensure the row was populated.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Set_ExistingDestination", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
// Set bits.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
}); err != nil {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
// Store row 10 into an existing row.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil {
t.Fatal(err)
} else if res := res.Results[0].(bool); !res {
t.Fatalf("unexpected set row result: %+v", res)
}
// Ensure the row was populated.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
}
func benchmarkExistence(nn bool, b *testing.B) {
c := test.MustRunCluster(b, 1)
defer c.Close()

View file

@ -52,6 +52,7 @@ const (
FieldTypeInt = "int"
FieldTypeTime = "time"
FieldTypeMutex = "mutex"
FieldTypeBool = "bool"
)
// Field represents a container for views.
@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption {
}
}
func OptFieldTypeBool() FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
fo.Type = FieldTypeBool
return nil
}
}
// NewField returns a new instance of field.
func NewField(path, index, name string, opts FieldOption) (*Field, error) {
err := validateName(name)
@ -219,8 +230,17 @@ func (f *Field) AvailableShards() *roaring.Bitmap {
return b
}
// addRemoteAvailableShards merges the set of available shards into the current known set.
func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) {
// addRemoteAvailableShards merges the set of available shards into the current known set
// and saves the set to a file.
func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error {
f.mergeRemoteAvailableShards(b)
// Save the updated bitmap to the data store.
return f.saveAvailableShards()
}
// mergeRemoteAvailableShards merges the set of available shards into the current known set.
func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
f.mu.Lock()
defer f.mu.Unlock()
f.remoteAvailableShards = f.remoteAvailableShards.Union(b)
@ -280,6 +300,10 @@ func (f *Field) Open() error {
return errors.Wrap(err, "loading meta")
}
if err := f.loadAvailableShards(); err != nil {
return errors.Wrap(err, "loading available shards")
}
// Apply the field options loaded from meta.
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
@ -441,6 +465,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Max = 0
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
case FieldTypeBool:
f.options.Type = FieldTypeBool
f.options.CacheType = CacheTypeNone
f.options.CacheSize = 0
f.options.Min = 0
f.options.Max = 0
f.options.TimeQuantum = ""
f.options.Keys = false
default:
return errors.New("invalid field type")
}
@ -448,6 +480,47 @@ func (f *Field) applyOptions(opt FieldOptions) error {
return nil
}
// loadAvailableShards reads remoteAvailableShards data for the field, if any.
func (f *Field) loadAvailableShards() error {
bm := roaring.NewBitmap()
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading available shards")
} else {
if err := bm.UnmarshalBinary(buf); err != nil {
return errors.Wrap(err, "unmarshaling")
}
}
// Merge bitmap from file into field.
f.mergeRemoteAvailableShards(bm)
return nil
}
// saveAvailableShards writes remoteAvailableShards data for the field.
func (f *Field) saveAvailableShards() error {
// Open or create file.
file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
return errors.Wrap(err, "opening available shards file")
}
f.mu.RLock()
defer f.mu.RUnlock()
// Write available shards to file.
if _, err := f.remoteAvailableShards.WriteTo(file); err != nil {
return errors.Wrap(err, "writing bitmap to buffer")
}
return nil
}
// Close closes the field and its views.
func (f *Field) Close() error {
f.mu.Lock()
@ -681,6 +754,10 @@ func (f *Field) deleteView(name string) error {
}
// Row returns a row of the standard view.
// It seems this method is only being used by the test
// package, and the fact that it's only allowed on
// `set` fields is odd. This may be considered for
// deprecation in a future version.
func (f *Field) Row(rowID uint64) (*Row, error) {
if f.Type() != FieldTypeSet {
return nil, errors.Errorf("row method unsupported for field type: %s", f.Type())
@ -954,10 +1031,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
return errors.New("time quantum not set in field")
}
fieldType := f.Type()
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
// Bool-specific data validation.
if fieldType == FieldTypeBool && rowID > 1 {
return errors.New("bool field imports only support values 0 and 1")
}
var timestamp *time.Time
if len(timestamps) > i {
timestamp = timestamps[i]
@ -1191,6 +1276,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
o.CacheSize,
o.Keys,
})
case FieldTypeBool:
return json.Marshal(struct {
Type string `json:"type"`
}{
o.Type,
})
}
return nil, errors.New("invalid field type")
}

View file

@ -22,6 +22,7 @@ import (
"time"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/roaring"
)
// Ensure a bsiGroup can adjust to its baseValue.
@ -341,3 +342,22 @@ func TestField_RowTime(t *testing.T) {
}
}
func TestField_PersistAvailableShards(t *testing.T) {
f := MustOpenField(OptFieldTypeDefault())
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
if err := f.addRemoteAvailableShards(bm); err != nil {
t.Fatal(err)
}
// Reload field and verify that shard data is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
}
}

View file

@ -75,6 +75,10 @@ const (
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 2000
// Row ids used for boolean fields.
falseRowID = uint64(0)
trueRowID = uint64(1)
)
// fragment represents the intersection of a field and shard in an index.
@ -351,11 +355,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row {
}
// Only use a subset of the containers.
// NOTE: The start & end ranges must be divisible by
// NOTE: The start & end ranges must be divisible by container width.
data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
// Reference bitmap subrange in storage.
// We Clone() data because otherwise row will contains pointers to containers in storage.
// We Clone() data because otherwise row will contain pointers to containers in storage.
// This causes unexpected results when we cache the row and try to use it later.
row := &Row{
segments: []rowSegment{{
@ -390,12 +394,13 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
// handleMutex will clear an existing row and store the new row
// in the vector.
func (f *fragment) handleMutex(rowID, columnID uint64) error {
if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID {
if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
return errors.Wrap(err, "getting mutex vector data")
} else if found && existingRowID != rowID {
if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil {
return errors.Wrap(err, "clearing mutex value")
}
}
f.mutexVector.Set(columnID, rowID)
return nil
}
@ -489,6 +494,95 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
return changed, nil
}
// setRow replaces an existing row (specified by rowID) with the given
// Row. This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.unprotectedSetRow(row, rowID)
}
func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) {
// TODO: In order to return `changed`, we need to first compare
// the existing row with the given row. Determine if the overhead
// of this is worth having `changed`.
// For now we will assume changed is always true.
changed = true
// First container of the row in storage.
headContainerKey := rowID << shardVsContainerExponent
// Remove every existing container in the row.
for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
f.storage.Containers.Remove(headContainerKey + i)
}
// From the given row, get the rowSegment for this shard.
seg := row.segment(f.shard)
if seg == nil {
return changed, nil
}
// Put each container from rowSegment to fragment storage.
citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent)
for citer.Next() {
k, c := citer.Value()
f.storage.Containers.Put(headContainerKey+(k%(1<<shardVsContainerExponent)), c)
}
// Update the row in cache.
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
f.cache.BulkAdd(rowID, n)
// Snapshot storage.
if err := f.snapshot(); err != nil {
return false, errors.Wrap(err, "snapshotting")
}
f.stats.Count("setRow", 1, 1.0)
return changed, nil
}
// ClearRow clears a row for a given rowID within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) clearRow(rowID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.unprotectedClearRow(rowID)
}
func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
changed = false
// First container of the row in storage.
headContainerKey := rowID << shardVsContainerExponent
// Remove every container in the row.
for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
k := headContainerKey + i
// Technically we could bypass the Get() call and only
// call Remove(), but the Get() gives us the ability
// to return true if any existing data was removed.
if cont := f.storage.Containers.Get(k); cont != nil {
f.storage.Containers.Remove(k)
changed = true
}
}
// Clear the row in cache.
f.cache.Add(rowID, 0)
// Snapshot storage.
if err := f.snapshot(); err != nil {
return false, errors.Wrap(err, "snapshotting")
}
f.stats.Count("clearRow", 1, 1.0)
return changed, nil
}
func (f *fragment) bit(rowID, columnID uint64) (bool, error) {
pos, err := f.pos(rowID, columnID)
if err != nil {
@ -1333,16 +1427,29 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs))
}
if f.mutexVector != nil {
return f.bulkImportMutex(rowIDs, columnIDs)
}
return f.bulkImportStandard(rowIDs, columnIDs)
}
// bulkImportStandard performs a bulk import on a standard fragment.
func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64) error {
// Create a temporary bitmap which will be populated by rowIDs and columnIDs
// and then merged into the existing fragment's bitmap.
localBitmap := roaring.NewBitmap()
// Disconnect op writer so we don't append updates.
localBitmap.OpWriter = nil
// Process every bit.
// If an error occurs then reopen the storage.
lastID := uint64(0)
// rowSet maintains the set of rowIDs present in this import.
// It allows the cache to be updated once per row, instead of once
// per bit.
rowSet := make(map[uint64]struct{})
lastRowID := uint64(0)
// Process every bit by writing to a local bitmap,
// to be merged with fragment storage next.
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
@ -1352,18 +1459,18 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
return err
}
// Write to storage.
// Write to local storage.
_, err = localBitmap.Add(pos)
if err != nil {
return err
}
// Reduce the StatsD rate for high volume stats
f.stats.Count("ImportBit", 1, 0.0001)
// import optimization to avoid linear foreach calls
// slight risk of concurrent cache counter being off but
// no real danger
if i == 0 || rowID != lastID {
lastID = rowID
// Add row to rowSet.
if i == 0 || rowID != lastRowID {
lastRowID = rowID
rowSet[rowID] = struct{}{}
}
@ -1392,6 +1499,98 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
return unprotectedWriteToFragment(f, results)
}
// bulkImportMutex performs a bulk import on a fragment while ensuring
// mutex restrictions. Because the mutex requirements must be checked
// against storage, this method must acquire a write lock on the fragment
// during the entire process, and it handles every bit independently.
func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
// Disconnect op writer so we don't append updates.
f.storage.OpWriter = nil
// If an error occurs then reopen the storage.
if err := func() error {
// rowSet maintains the set of rowIDs present in this import.
// It allows the cache to be updated once per row, instead of once
// per bit.
rowSet := make(map[uint64]struct{})
lastRowID := uint64(0)
// Process every bit.
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
// Handle mutex vector (i.e. clear an existing row).
if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
return errors.Wrap(err, "getting mutex vector data")
} else if found && existingRowID != rowID {
// Determine the position of the bit in the storage.
pos, err := f.pos(existingRowID, columnID)
if err != nil {
return err
}
// Clear storage.
_, err = f.storage.Remove(pos)
if err != nil {
return err
}
rowSet[existingRowID] = struct{}{}
}
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return err
}
// Write to storage.
_, err = f.storage.Add(pos)
if err != nil {
return err
}
// Reduce the StatsD rate for high volume stats
f.stats.Count("ImportBit", 1, 0.0001)
// Add row to rowSet.
if i == 0 || rowID != lastRowID {
lastRowID = rowID
rowSet[rowID] = struct{}{}
}
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
}
// Update cache counts for all rows.
for rowID := range rowSet {
// Import should ALWAYS have row() load a new bm from fragment.storage
// because the row that's in rowCache hasn't been updated with
// this import's data.
f.cache.BulkAdd(rowID, f.unprotectedRow(rowID).Count())
}
f.cache.Invalidate()
return nil
}(); err != nil {
_ = f.closeStorage()
_ = f.openStorage()
return err
}
// Write the storage to disk and reload.
if err := f.snapshot(); err != nil {
return err
}
return nil
}
// importValue bulk imports a set of range-encoded values.
func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error {
f.mu.Lock()
@ -2073,8 +2272,7 @@ func pos(rowID, columnID uint64) uint64 {
// vector stores the mapping of colID to rowID.
// It's used for a mutex field type.
type vector interface {
Get(colID uint64) (uint64, bool)
Set(colID, rowID uint64)
Get(colID uint64) (uint64, bool, error)
}
// rowsVector implements the vector interface by looking
@ -2093,20 +2291,50 @@ func newRowsVector(f *fragment) *rowsVector {
// Get returns the rowID associated to the given colID.
// Additionally, it returns true if a value was found,
// otherwise it returns false.
func (v *rowsVector) Get(colID uint64) (uint64, bool) {
func (v *rowsVector) Get(colID uint64) (uint64, bool, error) {
rows := v.f.rows(0, filterColumn(colID))
if len(rows) == 1 {
return rows[0], true
if len(rows) > 1 {
return 0, false, errors.New("found multiple row values for column")
} else if len(rows) == 1 {
return rows[0], true, nil
}
return 0, false
return 0, false, nil
}
// Set is not used for rowsVector.
func (v *rowsVector) Set(colID, rowID uint64) {}
// rowToKey converts a Pilosa row ID to the key of the container which starts
// that row in the bitmap which represents this entire fragment. A fragment is
// all the rows within a shard within a field concatenated together.
func rowToKey(rowID uint64) (key uint64) {
return rowID * (ShardWidth / containerWidth)
}
// boolVector implements the vector interface by looking
// at data in rows 0 and 1.
type boolVector struct {
f *fragment
}
// newBoolVector returns a boolVector for a given fragment.
func newBoolVector(f *fragment) *boolVector {
return &boolVector{
f: f,
}
}
// Get returns the rowID associated to the given colID.
// Additionally, it returns true if a value was found,
// otherwise it returns false.
func (v *boolVector) Get(colID uint64) (uint64, bool, error) {
rows := v.f.rows(0, filterColumn(colID))
if len(rows) > 1 {
return 0, false, errors.New("found multiple row values for column")
} else if len(rows) == 1 {
switch rows[0] {
case falseRowID, trueRowID:
return rows[0], true, nil
default:
return 0, false, errors.New("found non-boolean value")
}
}
return 0, false, nil
}

View file

@ -95,6 +95,81 @@ func TestFragment_ClearBit(t *testing.T) {
}
}
// Ensure a fragment can clear a row.
func TestFragment_ClearRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(1000, 65536); err != nil {
t.Fatal(err)
} else if _, err := f.unprotectedClearRow(1000); err != nil {
t.Fatal(err)
}
// Verify count on row.
if n := f.row(1000).Count(); n != 0 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.row(1000).Count(); n != 0 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
// Ensure a fragment can set a row.
func TestFragment_SetRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 7, "")
defer f.Close()
rowID := uint64(1000)
// Set bits on the fragment.
if _, err := f.setBit(rowID, 8000001); err != nil {
t.Fatal(err)
} else if _, err := f.setBit(rowID, 8065536); err != nil {
t.Fatal(err)
}
// Verify data on row.
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
t.Fatalf("unexpected columns: %+v", cols)
}
// Verify count on row.
if n := f.row(rowID).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
}
// Set row (overwrite existing data).
row := NewRow(8000002, 8065537, 8131074)
if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatalf("expected changed value: %v", changed)
}
// Verify data on row.
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
t.Fatalf("unexpected columns after set row: %+v", cols)
}
// Verify count on row.
if n := f.row(rowID).Count(); n != 3 {
t.Fatalf("unexpected count after set row: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.reopen(); err != nil {
t.Fatal(err)
} else if n := f.row(rowID).Count(); n != 3 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
// Ensure a fragment can set & read a value.
func TestFragment_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -1178,6 +1253,146 @@ func TestFragment_SetMutex(t *testing.T) {
}
}
// Ensure a fragment can import mutually exclusive values.
func TestFragment_ImportMutex(t *testing.T) {
tests := []struct {
rowIDs []uint64
colIDs []uint64
exp map[uint64][]uint64
}{
{
[]uint64{1, 1, 1, 1},
[]uint64{0, 1, 2, 3},
map[uint64][]uint64{
1: {0, 1, 2, 3},
},
},
{
[]uint64{1, 1, 1, 1, 2, 2, 2, 2},
[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
map[uint64][]uint64{
1: {},
2: {0, 1, 2, 3},
},
},
{
[]uint64{1, 1, 1, 1, 2},
[]uint64{0, 1, 2, 3, 1},
map[uint64][]uint64{
1: {0, 2, 3},
2: {1},
},
},
{
[]uint64{1, 1, 1, 1, 2, 2, 1},
[]uint64{0, 1, 2, 3, 1, 8, 1},
map[uint64][]uint64{
1: {0, 1, 2, 3},
2: {8},
},
},
{
[]uint64{1, 2, 3},
[]uint64{8, 8, 8},
map[uint64][]uint64{
1: {},
2: {},
3: {8},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
defer f.Close()
err := f.bulkImport(test.rowIDs, test.colIDs)
if err != nil {
t.Fatalf("bulk importing ids: %v", err)
}
// Check for expected results.
for k, v := range test.exp {
cols := f.row(k).Columns()
if !reflect.DeepEqual(cols, v) {
t.Fatalf("expected: %v, but got: %v", v, cols)
}
}
})
}
}
// Ensure a fragment can import bool values.
func TestFragment_ImportBool(t *testing.T) {
tests := []struct {
rowIDs []uint64
colIDs []uint64
exp map[uint64][]uint64
}{
{
[]uint64{1, 1, 1, 1},
[]uint64{0, 1, 2, 3},
map[uint64][]uint64{
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 0, 0, 0, 1, 1, 1, 1},
[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
map[uint64][]uint64{
0: {},
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 0, 0, 0, 1},
[]uint64{0, 1, 2, 3, 1},
map[uint64][]uint64{
0: {0, 2, 3},
1: {1},
},
},
{
[]uint64{1, 1, 1, 1, 0, 0, 1},
[]uint64{0, 1, 2, 3, 1, 8, 1},
map[uint64][]uint64{
0: {8},
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 1, 2},
[]uint64{8, 8, 8},
map[uint64][]uint64{
0: {},
1: {}, // This isn't {8} because fragment doesn't validate bool values.
2: {8},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
defer f.Close()
err := f.bulkImport(test.rowIDs, test.colIDs)
if err != nil {
t.Fatalf("bulk importing ids: %v", err)
}
// Check for expected results.
for k, v := range test.exp {
cols := f.row(k).Columns()
if !reflect.DeepEqual(cols, v) {
t.Fatalf("expected: %v, but got: %v", v, cols)
}
}
})
}
}
func BenchmarkFragment_Snapshot(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
@ -1302,6 +1517,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st
return frag
}
// mustOpenBoolFragment returns a new instance of Fragment for a bool field.
func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment {
frag := mustOpenFragment(index, field, view, shard, cacheType)
frag.mutexVector = newBoolVector(frag)
return frag
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *fragment) reopen() error {
if err := f.Close(); err != nil {

View file

@ -15,6 +15,8 @@
package gopsutil
import (
"runtime"
"github.com/pilosa/pilosa"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
@ -109,6 +111,11 @@ func (s *systemInfo) KernelVersion() (string, error) {
return host.KernelVersion()
}
// CPUArch returns the CPU architecture, such as amd64
func (s *systemInfo) CPUArch() string {
return runtime.GOARCH
}
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
func NewSystemInfo() *systemInfo {
return &systemInfo{}

View file

@ -15,7 +15,6 @@
package gopsutil_test
import (
"log"
"testing"
"github.com/pilosa/pilosa"
@ -25,15 +24,6 @@ import (
func TestSystemInfo(t *testing.T) {
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
// Uptime()(uint64, error)
// Platform()(string, error)
// Family()(string, error)
// OSVersion()(string, error)
// KernelVersion()(string, error)
// MemFree()(uint64, error)
// MemTotal()(uint64, error)
// MemUsed()(uint64, error)
//
uptime, err := systemInfo.Uptime()
if err != nil || uptime == 0 {
t.Fatalf("Error collecting uptime (error: %v)", err)
@ -70,8 +60,12 @@ func TestSystemInfo(t *testing.T) {
}
memtotal, err := systemInfo.MemTotal()
log.Println(memtotal)
if err != nil {
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
}
cpuArch := systemInfo.CPUArch()
if cpuArch == "" {
t.Fatalf("Error getting CPU arch.")
}
}

View file

@ -170,13 +170,34 @@ func (h *Handler) Close() error {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["Home"] = queryValidationSpecRequired()
h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired()
h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired()
h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired()
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetIndexes"] = queryValidationSpecRequired()
h.validators["GetIndex"] = queryValidationSpecRequired()
h.validators["PostIndex"] = queryValidationSpecRequired()
h.validators["DeleteIndex"] = queryValidationSpecRequired()
h.validators["PostField"] = queryValidationSpecRequired()
h.validators["DeleteField"] = queryValidationSpecRequired()
h.validators["PostImport"] = queryValidationSpecRequired()
h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetInfo"] = queryValidationSpecRequired()
h.validators["RecalculateCaches"] = queryValidationSpecRequired()
h.validators["GetSchema"] = queryValidationSpecRequired()
h.validators["GetStatus"] = queryValidationSpecRequired()
h.validators["GetVersion"] = queryValidationSpecRequired()
h.validators["PostClusterMessage"] = queryValidationSpecRequired()
h.validators["GetFragmentBlockData"] = queryValidationSpecRequired()
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired()
h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
h.validators["GetNodes"] = queryValidationSpecRequired()
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["GetTranslateData"] = queryValidationSpecRequired("offset")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -203,40 +224,40 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
// newRouter creates a new mux http router.
func newRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
router.HandleFunc("/", handler.handleHome).Methods("GET")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home")
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
// /internal endpoints are for internal use only; they may change at any time.
// DO NOT rely on these for external applications!
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
// TODO: Apply MethodNotAllowed statuses to all endpoints.
// Ideally this would be automatic, as described in this (wontfix) ticket:
@ -521,7 +542,12 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
return err
}
// Unmarshal expected values.
var _p _postIndexRequest
_p := _postIndexRequest{
Options: pilosa.IndexOptions{
Keys: false,
TrackExistence: true,
},
}
if err := json.Unmarshal(b, &_p); err != nil {
return errors.Wrap(err, "unmarshalling expected values")
}
@ -597,7 +623,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp := successResponse{}
// Decode request.
var req postIndexRequest
req := postIndexRequest{
Options: pilosa.IndexOptions{
Keys: false,
TrackExistence: true,
},
}
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil && err != io.EOF {
resp.write(w, err)
@ -687,6 +718,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
case pilosa.FieldTypeMutex:
fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize))
case pilosa.FieldTypeBool:
fos = append(fos, pilosa.OptFieldTypeBool())
}
if req.Options.Keys != nil {
if *req.Options.Keys {
@ -778,6 +811,20 @@ func (o *fieldOptions) validate() error {
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
}
case pilosa.FieldTypeBool:
if o.CacheType != nil {
return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool"))
} else if o.CacheSize != nil {
return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool"))
} else if o.Min != nil {
return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool"))
} else if o.Max != nil {
return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool"))
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
} else if o.Keys != nil {
return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
}
default:
return errors.Errorf("invalid field type: %s", o.Type)
}
@ -1457,11 +1504,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
return
}
resp := &pilosa.ImportResponse{}
// TODO give meaningful stats for import
err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body)
resp := &pilosa.ImportResponse{}
if err != nil {
resp.Err = err.Error()
if _, ok := err.(pilosa.BadRequestError); ok {
w.WriteHeader(http.StatusBadRequest)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
}
// Marshal response object.
buf, err := h.api.Serializer.Marshal(resp)

View file

@ -31,8 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
expected postIndexRequest
err string
}{
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}},
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}},
{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}},
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
@ -53,7 +54,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
if test.err == "" {
if !reflect.DeepEqual(*actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, *actual)
t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json)
}
}

View file

@ -69,9 +69,10 @@ func NewIndex(path, name string) (*Index, error) {
newAttrStore: newNopAttrStore,
columnAttrs: nopStore,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
logger: NopLogger,
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
logger: NopLogger,
trackExistence: true,
}, nil
}

View file

@ -119,7 +119,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
ID uint64 `json:"id,omitempty"`
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}

View file

@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) {
return "", fmt.Errorf("No field argument specified")
}
// BoolArg is for reading the value at key from call.Args as a bool. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a bool. An error is
// returned if the value is not a bool.
func (c *Call) BoolArg(key string) (bool, bool, error) {
val, ok := c.Args[key]
if !ok {
return false, false, nil
}
switch tval := val.(type) {
case bool:
return tval, true, nil
default:
return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval)
}
}
// UintArg is for reading the value at key from call.Args as a uint64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a uint64 or an int64 and

View file

@ -10,6 +10,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
/ 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
/ 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
/ 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
/ 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }

File diff suppressed because it is too large Load diff

View file

@ -51,13 +51,13 @@ func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n
if i < 0 {
c := NewContainer()
c.containerType = containerType
c.n = n
c.n = int32(n)
c.mapped = mapped
sc.insertAt(key, c, -i-1)
} else {
c := sc.containers[i]
c.containerType = containerType
c.n = n
c.n = int32(n)
c.mapped = mapped
}

View file

@ -46,7 +46,7 @@ const (
// at the beginning of every serialized run container.
runCountHeaderSize = 2
// interval32Size is the size of a single run in a container.runs.
// interval16Size is the size of a single run in a container.runs.
interval16Size = 4
// bitmapN is the number of values in a container.bitmap.
@ -73,6 +73,7 @@ type Containers interface {
// PutContainerValues updates an existing container at key.
// If a container does not exist for key, a new one is allocated.
// TODO(2.0) make n int32
PutContainerValues(key uint64, containerType byte, n int, mapped bool)
// Remove takes the container at key out.
@ -242,7 +243,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
if found && skey == ekey {
citer.Next()
_, c := citer.Value()
return uint64(c.countRange(int(lowbits(start)), int(lowbits(end))))
return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end))))
}
for citer.Next() {
@ -252,7 +253,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
panic("should be impossible for k to be less than skey")
}
if k == skey {
n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1))
n += uint64(c.countRange(int32(lowbits(start)), maxContainerVal+1))
continue
}
if k < ekey {
@ -260,7 +261,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
continue
}
if k == ekey {
n += uint64(c.countRange(0, int(lowbits(end))))
n += uint64(c.countRange(0, int32(lowbits(end))))
break
}
if k > ekey {
@ -805,7 +806,7 @@ type Iterator struct {
citer ContainerIterator
key uint64
c *Container
j, k int // i: container; j: array index, bit index, or run index; k: offset within the run
j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run
}
// Seek moves to the first value equal to or greater than `seek`.
@ -830,7 +831,7 @@ func (itr *Iterator) Seek(seek uint64) {
if itr.j < 0 {
itr.j = -itr.j - 1
}
if itr.j < len(itr.c.array) {
if itr.j < int32(len(itr.c.array)) {
itr.j--
return
}
@ -853,7 +854,7 @@ func (itr *Iterator) Seek(seek uint64) {
j, contains := binSearchRuns(lb, itr.c.runs)
if contains {
itr.j = j
itr.k = int(lb) - int(itr.c.runs[j].start) - 1
itr.k = int32(lb) - int32(itr.c.runs[j].start) - 1
} else {
// Set iterator to next value in the Bitmap.
itr.j = j
@ -864,7 +865,7 @@ func (itr *Iterator) Seek(seek uint64) {
}
// If it's a bitmap container then move to index before the value and call next().
itr.j = int(lb) - 1
itr.j = int32(lb) - 1
}
// Next returns the next value in the bitmap.
@ -914,14 +915,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
}
r := itr.c.runs[itr.j]
runLength := int(r.last - r.start)
runLength := int32(r.last - r.start)
if itr.k >= runLength {
// Reached end of run, move to the next run.
itr.j, itr.k = itr.j+1, -1
}
if itr.j >= len(itr.c.runs) {
if itr.j >= int32(len(itr.c.runs)) {
// Reached end of runs, move to the next container.
if !itr.citer.Next() {
itr.c = nil
@ -942,7 +943,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
// Find first non-zero bit in current bitmap, if possible.
hb := itr.j >> 6
if hb >= len(itr.c.bitmap) {
if hb >= int32(len(itr.c.bitmap)) {
if !itr.citer.Next() {
itr.c = nil
return 0, true
@ -953,14 +954,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
}
lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64)
if lb != 0 {
itr.j = itr.j + trailingZeroN(lb)
itr.j = itr.j + int32(trailingZeroN(lb))
return itr.peek(), false
}
// Otherwise iterate through remaining bitmaps to find next bit.
for hb++; hb < len(itr.c.bitmap); hb++ {
for hb++; hb < int32(len(itr.c.bitmap)); hb++ {
if itr.c.bitmap[hb] != 0 {
itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb])
itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap[hb]))
return itr.peek(), false
}
}
@ -1008,7 +1009,7 @@ const runMaxSize = 2048
type Container struct {
mapped bool // mapped directly to a byte slice when true
containerType byte // array, bitmap, or run
n int // number of integers in container
n int32 // number of integers in container
array []uint16 // used for array containers
bitmap []uint64 // used for bitmap containers
runs []interval16 // used for RLE containers
@ -1020,8 +1021,8 @@ type interval16 struct {
}
// runlen returns the count of integers in the interval.
func (iv interval16) runlen() int {
return 1 + int(iv.last-iv.start)
func (iv interval16) runlen() int32 {
return 1 + int32(iv.last-iv.start)
}
// newContainer returns a new instance of container.
@ -1035,12 +1036,12 @@ func (c *Container) Mapped() bool {
}
// N returns the cached bit count of the container
func (c *Container) N() int {
func (c *Container) N() int32 {
return c.n
}
// Update updates the container
func (c *Container) Update(containerType byte, n int, mapped bool) {
func (c *Container) Update(containerType byte, n int32, mapped bool) {
c.containerType = containerType
c.n = n
c.mapped = mapped
@ -1088,12 +1089,12 @@ func (c *Container) unmap() {
}
// count counts all bits in the container.
func (c *Container) count() (n int) {
func (c *Container) count() (n int32) {
return c.countRange(0, maxContainerVal+1)
}
// countRange counts the number of bits set between [start, end).
func (c *Container) countRange(start, end int) (n int) {
func (c *Container) countRange(start, end int32) (n int32) {
if c.isArray() {
return c.arrayCountRange(start, end)
} else if c.isRun() {
@ -1102,10 +1103,10 @@ func (c *Container) countRange(start, end int) (n int) {
return c.bitmapCountRange(start, end)
}
func (c *Container) arrayCountRange(start, end int) (n int) {
i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start })
for ; i < len(c.array); i++ {
v := int(c.array[i])
func (c *Container) arrayCountRange(start, end int32) (n int32) {
i := int32(sort.Search(len(c.array), func(i int) bool { return int32(c.array[i]) >= start }))
for ; i < int32(len(c.array)); i++ {
v := int32(c.array[i])
if v >= end {
break
}
@ -1114,14 +1115,14 @@ func (c *Container) arrayCountRange(start, end int) (n int) {
return n
}
func (c *Container) bitmapCountRange(start, end int) int {
func (c *Container) bitmapCountRange(start, end int32) int32 {
var n uint64
i, j := start/64, end/64
// Special case when start and end fall in the same word.
if i == j {
offi, offj := uint(start%64), uint(64-end%64)
n += popcount((c.bitmap[i] >> offi) << (offj + offi))
return int(n)
return int32(n)
}
// Count partial starting word.
@ -1136,39 +1137,39 @@ func (c *Container) bitmapCountRange(start, end int) int {
}
// Count partial ending word.
if j < len(c.bitmap) {
if j < int32(len(c.bitmap)) {
off := 64 - (uint(end) % 64)
n += popcount(c.bitmap[j] << off)
}
return int(n)
return int32(n)
}
func (c *Container) runCountRange(start, end int) (n int) {
func (c *Container) runCountRange(start, end int32) (n int32) {
for _, iv := range c.runs {
// iv is before range
if int(iv.last) < start {
if int32(iv.last) < start {
continue
}
// iv is after range
if end < int(iv.start) {
if end < int32(iv.start) {
break
}
// iv is superset of range
if int(iv.start) < start && int(iv.last) > end {
if int32(iv.start) < start && int32(iv.last) > end {
return end - start
}
// iv is subset of range
if int(iv.start) >= start && int(iv.last) < end {
if int32(iv.start) >= start && int32(iv.last) < end {
n += iv.runlen()
}
// iv overlaps beginning of range
if int(iv.start) < start && int(iv.last) < end {
n += int(iv.last) - start + 1
if int32(iv.start) < start && int32(iv.last) < end {
n += int32(iv.last) - start + 1
}
// iv overlaps end of range
if int(iv.start) > start && int(iv.last) >= end {
n += end - int(iv.start)
if int32(iv.start) > start && int32(iv.last) >= end {
n += end - int32(iv.start)
}
}
return n
@ -1286,34 +1287,34 @@ func (c *Container) Contains(v uint16) bool {
}
}
func (c *Container) bitmapCountRuns() (r int) {
func (c *Container) bitmapCountRuns() (r int32) {
for i := 0; i < 1023; i++ {
v, v1 := c.bitmap[i], c.bitmap[i+1]
r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1))
r = r + int32(popcount((v<<1)&^v)+((v>>63)&^v1))
}
vl := c.bitmap[len(c.bitmap)-1]
r = r + int(popcount((vl<<1)&^vl)+vl>>63)
r = r + int32(popcount((vl<<1)&^vl)+vl>>63)
return r
}
func (c *Container) arrayCountRuns() (r int) {
prev := -2
func (c *Container) arrayCountRuns() (r int32) {
prev := int32(-2)
for _, v := range c.array {
if prev+1 != int(v) {
if prev+1 != int32(v) {
r++
}
prev = int(v)
prev = int32(v)
}
return r
}
func (c *Container) countRuns() (r int) {
func (c *Container) countRuns() (r int32) {
if c.isArray() {
return c.arrayCountRuns()
} else if c.isBitmap() {
return c.bitmapCountRuns()
} else if c.isRun() {
return len(c.runs)
return int32(len(c.runs))
}
// sure hope this never happens
@ -1369,10 +1370,10 @@ func (c *Container) bitmapContains(v uint16) bool {
// binSearchRuns returns the index of the run containing v, and true, when v is contained;
// or the index of the next run starting after v, and false, when v is not contained.
func binSearchRuns(v uint16, a []interval16) (int, bool) {
i := sort.Search(len(a),
func(i int) bool { return a[i].last >= v })
if i < len(a) {
func binSearchRuns(v uint16, a []interval16) (int32, bool) {
i := int32(sort.Search(len(a),
func(i int) bool { return a[i].last >= v }))
if i < int32(len(a)) {
return i, (v >= a[i].start) && (v <= a[i].last)
}
@ -1698,7 +1699,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
// assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
//}
// Write sizeof(uint32) * cardinality bytes.
// Write sizeof(uint16) * cardinality bytes.
nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n])
return int64(nn), err
}
@ -1767,7 +1768,7 @@ func (c *Container) check() error {
var a ErrorList
if c.isArray() {
if len(c.array) != c.n {
if int32(len(c.array)) != c.n {
a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n))
}
} else if c.isRun() {
@ -1796,7 +1797,7 @@ func (c *Container) check() error {
type containerInfo struct {
Key uint64 // container key
Type string // container type (array, bitmap, or run)
N int // number of bits
N int32 // number of bits
Alloc int // memory used
Pointer unsafe.Pointer // offset within the mmap
}
@ -1838,7 +1839,7 @@ func flipRun(b *Container) *Container {
return flipBitmap(x)
}
func intersectionCount(a, b *Container) int {
func intersectionCount(a, b *Container) int32 {
if a.isArray() {
if b.isArray() {
return intersectionCountArrayArray(a, b)
@ -1866,7 +1867,7 @@ func intersectionCount(a, b *Container) int {
}
}
func intersectionCountArrayArray(a, b *Container) (n int) {
func intersectionCountArrayArray(a, b *Container) (n int32) {
na, nb := len(a.array), len(b.array)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.array[j]
@ -1882,7 +1883,7 @@ func intersectionCountArrayArray(a, b *Container) (n int) {
return n
}
func intersectionCountArrayRun(a, b *Container) (n int) {
func intersectionCountArrayRun(a, b *Container) (n int32) {
na, nb := len(a.array), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.array[i], b.runs[j]
@ -1898,7 +1899,7 @@ func intersectionCountArrayRun(a, b *Container) (n int) {
return n
}
func intersectionCountRunRun(a, b *Container) (n int) {
func intersectionCountRunRun(a, b *Container) (n int32) {
na, nb := len(a.runs), len(b.runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := a.runs[i], b.runs[j]
@ -1910,33 +1911,33 @@ func intersectionCountRunRun(a, b *Container) (n int) {
j++
} else if va.last > vb.last && va.start >= vb.start {
// |--vb-|-|-va--|
n += 1 + int(vb.last-va.start)
n += 1 + int32(vb.last-va.start)
j++
} else if va.last > vb.last && va.start < vb.start {
// |--va|--vb--|--|
n += 1 + int(vb.last-vb.start)
n += 1 + int32(vb.last-vb.start)
j++
} else if va.last <= vb.last && va.start >= vb.start {
// |--vb|--va--|--|
n += 1 + int(va.last-va.start)
n += 1 + int32(va.last-va.start)
i++
} else if va.last <= vb.last && va.start < vb.start {
// |--va-|-|-vb--|
n += 1 + int(va.last-vb.start)
n += 1 + int32(va.last-vb.start)
i++
}
}
return n
}
func intersectionCountBitmapRun(a, b *Container) (n int) {
func intersectionCountBitmapRun(a, b *Container) (n int32) {
for _, iv := range b.runs {
n += a.bitmapCountRange(int(iv.start), int(iv.last)+1)
n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1)
}
return n
}
func intersectionCountArrayBitmap(a, b *Container) (n int) {
func intersectionCountArrayBitmap(a, b *Container) (n int32) {
ln := len(b.bitmap)
for _, val := range a.array {
i := int(val >> 6)
@ -1944,13 +1945,13 @@ func intersectionCountArrayBitmap(a, b *Container) (n int) {
break
}
off := val % 64
n += int(b.bitmap[i]>>off) & 1
n += int32(b.bitmap[i]>>off) & 1
}
return n
}
func intersectionCountBitmapBitmap(a, b *Container) (n int) {
return int(popcountAndSlice(a.bitmap, b.bitmap))
func intersectionCountBitmapBitmap(a, b *Container) (n int32) {
return int32(popcountAndSlice(a.bitmap, b.bitmap))
}
func intersect(a, b *Container) *Container {
@ -1995,7 +1996,7 @@ func intersectArrayArray(a, b *Container) *Container {
i, j = i+1, j+1
}
}
output.n = len(output.array)
output.n = int32(len(output.array))
return output
}
@ -2016,7 +2017,7 @@ func intersectArrayRun(a, b *Container) *Container {
i++
}
}
output.n = len(output.array)
output.n = int32(len(output.array))
return output
}
@ -2050,7 +2051,7 @@ func intersectRunRun(a, b *Container) *Container {
i++
}
}
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
output.runToArray()
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
@ -2076,7 +2077,7 @@ func intersectBitmapRun(a, b *Container) *Container {
}
}
}
output.n = len(output.array)
output.n = int32(len(output.array))
} else {
// right now this iterates through the runs and sets integers in the
// bitmap that are in the runs. alternately, we could zero out ranges in
@ -2093,22 +2094,22 @@ func intersectBitmapRun(a, b *Container) *Container {
for valast >= vb.start && vastart <= vb.last && i < bitmapN {
if vastart >= vb.start && valast <= vb.last { // a within b
output.bitmap[i] = a.bitmap[i]
output.n += int(popcount(a.bitmap[i]))
output.n += int32(popcount(a.bitmap[i]))
} else if vb.start >= vastart && vb.last <= valast { // b within a
var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
bits := a.bitmap[i] & mask
output.bitmap[i] |= bits
output.n += int(popcount(bits))
output.n += int32(popcount(bits))
} else if vastart < vb.start { // a overlaps front of b
offset := 64 - (1 + valast - vb.start)
bits := (a.bitmap[i] >> offset) << offset
output.bitmap[i] |= bits
output.n += int(popcount(bits))
output.n += int32(popcount(bits))
} else if vb.start < vastart { // b overlaps front of a
offset := 64 - (1 + vb.last - vastart)
bits := (a.bitmap[i] << offset) >> offset
output.bitmap[i] |= bits
output.n += int(popcount(bits))
output.n += int32(popcount(bits))
}
// update loop vars
i++
@ -2134,20 +2135,30 @@ func intersectArrayBitmap(a, b *Container) *Container {
output.array = append(output.array, va)
}
}
output.n = len(output.array)
output.n = int32(len(output.array))
return output
}
func intersectBitmapBitmap(a, b *Container) *Container {
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
for i := range a.bitmap {
v := a.bitmap[i] & b.bitmap[i]
output.bitmap[i] = v
output.n += int(popcount(v))
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap[:bitmapN]
bb = b.bitmap[:bitmapN]
buf = make([]uint64, bitmapN)
ob = buf[:bitmapN]
n int32
)
for i := 0; i < bitmapN; i++ {
ob[i] = ab[i] & bb[i]
n += int32(popcount(ob[i]))
}
output := &Container{
bitmap: ob,
n: n,
containerType: containerBitmap,
}
output.optimize()
return output
}
@ -2249,10 +2260,10 @@ func unionArrayRun(a, b *Container) *Container {
// interval is earlier than the start of the last interval in the list of runs.
// Its return value is the amount by which the cardinality of the container was
// increased.
func (c *Container) runAppendInterval(v interval16) int {
func (c *Container) runAppendInterval(v interval16) int32 {
if len(c.runs) == 0 {
c.runs = append(c.runs, v)
return int(v.last-v.start) + 1
return int32(v.last-v.start) + 1
}
last := c.runs[len(c.runs)-1]
@ -2261,10 +2272,10 @@ func (c *Container) runAppendInterval(v interval16) int {
}
if last.last+1 >= v.start && v.last > last.last {
c.runs[len(c.runs)-1].last = v.last
return int(v.last - last.last)
return int32(v.last - last.last)
} else if last.last+1 < v.start {
c.runs = append(c.runs, v)
return int(v.last-v.start) + 1
return int32(v.last-v.start) + 1
}
return 0
}
@ -2328,16 +2339,16 @@ func (c *Container) bitmapSetRange(i, j uint64) {
xcnt := popcount(X)
ycnt := popcount(Y)
if x == y {
c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y)))
c.n += int32((j - i) - popcount(c.bitmap[x]&(X&Y)))
c.bitmap[x] |= (X & Y)
} else {
c.n += int(xcnt - popcount(c.bitmap[x]&X))
c.n += int32(xcnt - popcount(c.bitmap[x]&X))
c.bitmap[x] |= X
for i := x + 1; i < y; i++ {
c.n += int(64 - popcount(c.bitmap[i]))
c.n += int32(64 - popcount(c.bitmap[i]))
c.bitmap[i] = maxBitmap
}
c.n += int(ycnt - popcount(c.bitmap[y]&Y))
c.n += int32(ycnt - popcount(c.bitmap[y]&Y))
c.bitmap[y] |= Y
}
}
@ -2351,19 +2362,19 @@ func (c *Container) bitmapXorRange(i, j uint64) {
if x == y {
cnt := popcount(c.bitmap[x])
c.bitmap[x] ^= (X & Y) //// flip
c.n += int(popcount(c.bitmap[x]) - cnt)
c.n += int32(popcount(c.bitmap[x]) - cnt)
} else {
cnt := popcount(c.bitmap[x])
c.bitmap[x] ^= X
c.n += int(popcount(c.bitmap[x]) - cnt)
c.n += int32(popcount(c.bitmap[x]) - cnt)
for i := x + 1; i < y; i++ {
cnt = popcount(c.bitmap[i])
c.bitmap[i] ^= maxBitmap
c.n += int(popcount(c.bitmap[i]) - cnt)
c.n += int32(popcount(c.bitmap[i]) - cnt)
}
cnt = popcount(c.bitmap[y])
c.bitmap[y] ^= Y
c.n += int(popcount(c.bitmap[y]) - cnt)
c.n += int32(popcount(c.bitmap[y]) - cnt)
}
}
@ -2374,16 +2385,16 @@ func (c *Container) bitmapZeroRange(i, j uint64) {
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
c.n -= int(popcount(c.bitmap[x] & (X & Y)))
c.n -= int32(popcount(c.bitmap[x] & (X & Y)))
c.bitmap[x] &= ^(X & Y)
} else {
c.n -= int(popcount(c.bitmap[x] & X))
c.n -= int32(popcount(c.bitmap[x] & X))
c.bitmap[x] &= ^X
for i := x + 1; i < y; i++ {
c.n -= int(popcount(c.bitmap[i]))
c.n -= int32(popcount(c.bitmap[i]))
c.bitmap[i] = 0
}
c.n -= int(popcount(c.bitmap[y] & Y))
c.n -= int32(popcount(c.bitmap[y] & Y))
c.bitmap[y] &= ^Y
}
}
@ -2437,17 +2448,28 @@ func unionArrayBitmap(a, b *Container) *Container {
}
func unionBitmapBitmap(a, b *Container) *Container {
output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: containerBitmap,
}
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap[:bitmapN]
bb = b.bitmap[:bitmapN]
buf = make([]uint64, bitmapN)
ob = buf[:bitmapN]
n int32
)
for i := 0; i < bitmapN; i++ {
v := a.bitmap[i] | b.bitmap[i]
output.bitmap[i] = v
output.n += int(popcount(v))
ob[i] = ab[i] | bb[i]
n += int32(popcount(ob[i]))
}
output := &Container{
bitmap: ob,
n: n,
containerType: containerBitmap,
}
return output
}
@ -2519,7 +2541,7 @@ func differenceArrayRun(a, b *Container) *Container {
j := 0 // run index
// handle overlap
for i < a.n {
for i < int(a.n) {
// keep all array elements before beginning of runs
if a.array[i] < b.runs[j].start {
@ -2551,7 +2573,7 @@ func differenceArrayRun(a, b *Container) *Container {
output.array = append(output.array, a.array[i:]...)
// TODO: consider handling container.n mutations in one place
// like we do with container.add().
output.n += len(a.array[i:])
output.n += int32(len(a.array[i:]))
} else {
for _, v := range a.array[i:] {
output.add(v)
@ -2609,7 +2631,7 @@ RUNLOOP:
continue
}
output.runs = append(output.runs, interval16{start: start, last: vb - 1})
output.n += int(vb - start)
output.n += int32(vb - start)
if vb == 65535 { // overflow
break RUNLOOP
}
@ -2623,7 +2645,7 @@ RUNLOOP:
if start <= run.last {
output.runs = append(output.runs, interval16{start: start, last: run.last})
output.n += int(run.last - start + 1)
output.n += int32(run.last - start + 1)
}
}
output.optimize()
@ -2679,7 +2701,7 @@ func differenceRunBitmap(a, b *Container) *Container {
}
}
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
output.runToArray()
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
@ -2763,7 +2785,7 @@ func differenceArrayBitmap(a, b *Container) *Container {
output.array = append(output.array, va)
}
}
output.n = len(output.array)
output.n = int32(len(output.array))
return output
}
@ -2783,13 +2805,27 @@ func differenceBitmapArray(a, b *Container) *Container {
}
func differenceBitmapBitmap(a, b *Container) *Container {
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
for i := range a.bitmap {
v := a.bitmap[i] & (^b.bitmap[i])
output.bitmap[i] = v
output.n += int(popcount(v))
var (
ab = a.bitmap[:bitmapN]
bb = b.bitmap[:bitmapN]
buf = make([]uint64, bitmapN)
ob = buf[:bitmapN]
n int32
)
for i := 0; i < bitmapN; i++ {
ob[i] = ab[i] & (^bb[i])
n += int32(popcount(ob[i]))
}
output := &Container{
bitmap: ob,
n: n,
containerType: containerBitmap,
}
if output.n < ArrayMaxSize {
output.bitmapToArray()
@ -2874,16 +2910,28 @@ func xorArrayBitmap(a, b *Container) *Container {
}
func xorBitmapBitmap(a, b *Container) *Container {
output := &Container{
bitmap: make([]uint64, bitmapN),
containerType: containerBitmap,
}
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap[:bitmapN]
bb = b.bitmap[:bitmapN]
buf = make([]uint64, bitmapN)
ob = buf[:bitmapN]
n int32
)
for i := 0; i < bitmapN; i++ {
v := a.bitmap[i] ^ b.bitmap[i]
output.bitmap[i] = v
output.n += int(popcount(v))
ob[i] = ab[i] ^ bb[i]
n += int32(popcount(ob[i]))
}
output := &Container{
bitmap: ob,
n: n,
containerType: containerBitmap,
}
if output.count() < ArrayMaxSize {
output.bitmapToArray()
}
@ -2962,9 +3010,9 @@ func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
// search32 returns the index of value in a. If value is not found, it works the
// same way as search64.
func search32(a []uint16, value uint16) int {
func search32(a []uint16, value uint16) int32 {
// Optimize for elements and the last element.
n := len(a)
n := int32(len(a))
if n == 0 {
return -1
} else if a[n-1] == value {
@ -2972,9 +3020,9 @@ func search32(a []uint16, value uint16) int {
}
// Otherwise perform binary search for exact match.
lo, hi := 0, n-1
lo, hi := int32(0), n-1
for lo+16 <= hi {
i := int(uint((lo + hi)) >> 1)
i := int32(uint((lo + hi)) >> 1)
v := a[i]
if v < value {
@ -3280,7 +3328,7 @@ func xorRunRun(a, b *Container) *Container {
}
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
output.runToArray()
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
@ -3295,7 +3343,7 @@ func xorBitmapRun(a, b *Container) *Container {
output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
}
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
output.runToArray()
} else if len(output.runs) > runMaxSize {
output.runToBitmap()
@ -3336,9 +3384,14 @@ func popcount(x uint64) uint64 {
}
func popcountAndSlice(s, m []uint64) uint64 {
var (
a = s[:bitmapN]
b = m[:bitmapN]
)
cnt := uint64(0)
for i := range s {
cnt += popcount(s[i] & m[i])
for i := 0; i < bitmapN; i++ {
cnt += popcount(a[i] & b[i])
}
return cnt
}
@ -3494,7 +3547,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) {
c.runs = nil
c.bitmap = nil
c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n]
pos += c.n * 2
pos += int(c.n * 2)
case containerBitmap:
c.array = nil
c.runs = nil

View file

@ -39,7 +39,7 @@ func TestRunAppendInterval(t *testing.T) {
tests := []struct {
base []interval16
app interval16
exp int
exp int32
}{
{
base: []interval16{},
@ -207,10 +207,10 @@ func TestRunContains(t *testing.T) {
func TestBitmapCountRange(t *testing.T) {
c := Container{containerType: containerBitmap}
tests := []struct {
start int
end int
start int32
end int32
bitmap []uint64
exp int
exp int32
}{
{start: 0, end: 1, bitmap: []uint64{1}, exp: 1},
{start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2},
@ -251,7 +251,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) {
b.bitmapToRun()
res = intersectRunRun(a, b)
n := intersectionCountRunRun(a, b)
if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) {
if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) {
t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1)
}
}
@ -261,7 +261,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
tests := []struct {
array []uint16
bitmap []uint64
exp int
exp int32
}{
{
array: []uint16{0},
@ -384,7 +384,7 @@ func TestIntersectionCountRunRun(t *testing.T) {
tests := []struct {
aruns []interval16
bruns []interval16
exp int
exp int32
}{
{
aruns: []interval16{},
@ -478,7 +478,7 @@ func TestIntersectRunRun(t *testing.T) {
aruns []interval16
bruns []interval16
exp []interval16
expN int
expN int32
}{
{
aruns: []interval16{},
@ -540,7 +540,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
bitmap []uint64
runs []interval16
exp []uint64
expN int
expN int32
}{
{
bitmap: []uint64{1},
@ -602,7 +602,7 @@ func TestIntersectBitmapRunArray(t *testing.T) {
bitmap []uint64
runs []interval16
exp []uint16
expN int
expN int32
}{
{
bitmap: []uint64{1},
@ -761,11 +761,11 @@ func TestDifferenceMixed(t *testing.T) {
a.containerType = containerRun
b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
b.n = len(b.array)
b.n = int32(len(b.array))
b.containerType = containerArray
d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
d.n = len(d.array)
d.n = int32(len(d.array))
d.containerType = containerArray
res := difference(a, b)
@ -784,11 +784,11 @@ func TestDifferenceMixed(t *testing.T) {
t.Fatalf("test #3 expected empty but got %v", res.runs)
}
c.bitmap = []uint64{0x64}
c.bitmap = MakeBitmap([]uint64{0x64})
c.n = c.countRange(0, 100)
c.containerType = containerBitmap
res = difference(c, a)
if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
if !reflect.DeepEqual(res.bitmap, MakeBitmap([]uint64{0x4})) {
t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
}
@ -939,7 +939,7 @@ func TestBitmapSetRange(t *testing.T) {
start uint64
last uint64
exp []uint64
expN int
expN int32
}{
{
bitmap: []uint64{0x0000000000FFF900},
@ -991,7 +991,7 @@ func TestArrayToBitmap(t *testing.T) {
copy(exp, test.exp)
a.array = test.array
a.n = len(test.array)
a.n = int32(len(test.array))
a.arrayToBitmap()
if !reflect.DeepEqual(a.bitmap, exp) {
t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
@ -1016,10 +1016,10 @@ func TestBitmapToArray(t *testing.T) {
}
for i, test := range tests {
a.bitmap = make([]uint64, bitmapN)
n := 0
n := int32(0)
for i, v := range test.bitmap {
a.bitmap[i] = v
n += int(popcount(v))
n += int32(popcount(v))
}
a.n = n
@ -1067,7 +1067,7 @@ func TestRunToBitmap(t *testing.T) {
}
a.runs = test.runs
a.n = n
a.n = int32(n)
a.runToBitmap()
if !reflect.DeepEqual(a.bitmap, exp) {
t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
@ -1149,7 +1149,7 @@ func TestBitmapToRun(t *testing.T) {
a.bitmap[i] = v
n += int(popcount(v))
}
a.n = n
a.n = int32(n)
x := a.bitmap
a.bitmapToRun()
if !reflect.DeepEqual(a.runs, test.exp) {
@ -1188,7 +1188,7 @@ func TestArrayToRun(t *testing.T) {
for i, test := range tests {
a.array = test.array
a.n = int(len(test.array))
a.n = int32(len(test.array))
a.arrayToRun()
if !reflect.DeepEqual(a.runs, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
@ -1222,7 +1222,7 @@ func TestRunToArray(t *testing.T) {
for i, test := range tests {
a.runs = test.runs
a.n = len(test.exp)
a.n = int32(len(test.exp))
a.runToArray()
if !reflect.DeepEqual(a.array, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
@ -1237,7 +1237,7 @@ func TestBitmapZeroRange(t *testing.T) {
start uint64
last uint64
exp []uint64
expN int
expN int32
}{
{
bitmap: []uint64{0x0000000000FFFF00},
@ -1279,7 +1279,7 @@ func TestUnionBitmapRun(t *testing.T) {
bitmap []uint64
runs []interval16
exp []uint64
expN int
expN int32
}{
{
bitmap: []uint64{2},
@ -1313,7 +1313,7 @@ func TestBitmapCountRuns(t *testing.T) {
c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
exp int
exp int32
}{
{
bitmap: []uint64{0xFF00FF00},
@ -1361,7 +1361,7 @@ func TestArrayCountRuns(t *testing.T) {
c := &Container{containerType: containerArray}
tests := []struct {
array []uint16
exp int
exp int32
}{
{
array: []uint16{},
@ -1414,7 +1414,7 @@ func TestDifferenceArrayRun(t *testing.T) {
}
for i, test := range tests {
a.array = test.array
a.n = len(a.array)
a.n = int32(len(a.array))
b.runs = test.runs
b.n = b.runCountRange(0, 100)
ret := differenceArrayRun(a, b)
@ -1482,7 +1482,7 @@ func TestDifferenceRunArray(t *testing.T) {
a.runs = test.runs
a.n = a.runCountRange(0, 100)
b.array = test.array
b.n = len(b.array)
b.n = int32(len(b.array))
ret := differenceRunArray(a, b)
if !reflect.DeepEqual(ret.runs, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@ -1730,7 +1730,7 @@ func TestDifferenceRunRun(t *testing.T) {
aruns []interval16
bruns []interval16
exp []interval16
expn int
expn int32
}{
{
// this tests all six overlap combinations
@ -2009,7 +2009,7 @@ func TestBitmapXorRange(t *testing.T) {
start uint64
last uint64
exp []uint64
expN int
expN int32
}{
{
bitmap: []uint64{0x0000000000000000},
@ -2302,7 +2302,7 @@ func TestRunBinSearchContains(t *testing.T) {
runs []interval16
index uint16
exp struct {
index int
index int32
found bool
}
}{
@ -2310,7 +2310,7 @@ func TestRunBinSearchContains(t *testing.T) {
runs: []interval16{{start: 0, last: 10}},
index: uint16(3),
exp: struct {
index int
index int32
found bool
}{index: 0, found: true},
},
@ -2318,7 +2318,7 @@ func TestRunBinSearchContains(t *testing.T) {
runs: []interval16{{start: 0, last: 10}},
index: uint16(13),
exp: struct {
index int
index int32
found bool
}{index: 0, found: false},
},
@ -2326,7 +2326,7 @@ func TestRunBinSearchContains(t *testing.T) {
runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
index: uint16(13),
exp: struct {
index int
index int32
found bool
}{index: 0, found: false},
},
@ -2334,7 +2334,7 @@ func TestRunBinSearchContains(t *testing.T) {
runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
index: uint16(36),
exp: struct {
index int
index int32
found bool
}{index: 1, found: false},
},
@ -2355,7 +2355,7 @@ func TestRunBinSearch(t *testing.T) {
runs []interval16
search uint16
exp bool
expi int
expi int32
}{
{
runs: []interval16{{2, 10}, {50, 60}, {80, 90}},

View file

@ -234,6 +234,13 @@ func OptServerClusterHasher(h Hasher) ServerOption {
}
}
func OptServerTranslateFileMapSize(mapSize int) ServerOption {
return func(s *Server) error {
s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize))
return nil
}
}
// NewServer returns a new instance of Server.
func NewServer(opts ...ServerOption) (*Server, error) {
s := &Server{
@ -330,20 +337,20 @@ func (s *Server) Open() error {
// Initialize id-key storage.
if err := s.holder.translateFile.Open(); err != nil {
return err
return errors.Wrap(err, "opening TranslateFile")
}
// Open Cluster management.
if err := s.cluster.waitForStarted(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
return errors.Wrap(err, "opening Cluster")
}
// Open holder.
if err := s.holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
return errors.Wrap(err, "opening Holder")
}
if err := s.cluster.setNodeState(nodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
return errors.Wrap(err, "setting nodeState")
}
// Listen for joining nodes.
@ -474,7 +481,9 @@ func (s *Server) receiveMessage(m Message) error {
if f == nil {
return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field)
}
f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard))
if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
case *CreateIndexMessage:
opt := obj.Meta
_, err := s.holder.CreateIndex(obj.Index, *opt)
@ -633,13 +642,15 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
for _, fs := range is.Fields {
f := s.holder.Field(is.Name, fs.Name)
// if we don't know about an field locally, log a error because
// if we don't know about a field locally, log an error because
// fields should be created and synced prior to shard creation
if f == nil {
s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name)
continue
}
f.addRemoteAvailableShards(fs.AvailableShards)
if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
}
}
@ -664,6 +675,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.nodeID)
s.diagnostics.Set("ClusterID", s.cluster.id)
s.diagnostics.EnrichWithCPUInfo()
s.diagnostics.EnrichWithOSInfo()
// Flush the diagnostics metrics at startup, then on each tick interval

View file

@ -71,8 +71,9 @@ type Config struct {
// Gossip config is based around memberlist.Config.
Gossip gossip.Config `toml:"gossip"`
// DEPRECATED: Translation config supports translation store replication.
Translation struct {
MapSize int `toml:"map-size"`
// DEPRECATED: Translation config supports translation store replication.
PrimaryURL string `toml:"primary-url"`
} `toml:"translation"`

View file

@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) {
})
t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) {
// Roaring import into a non-set field should fail.
if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData))
req.Header.Set("Content-Type", "application/x-binary")
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Status", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))

View file

@ -283,6 +283,14 @@ func (m *Command) SetupServer() error {
coordinatorOpt,
}
if m.Config.Translation.MapSize > 0 {
serverOptions = append(
serverOptions,
pilosa.OptServerTranslateFileMapSize(
m.Config.Translation.MapSize,
),
)
}
serverOptions = append(serverOptions, m.serverOptions...)
m.Server, err = pilosa.NewServer(serverOptions...)

View file

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"reflect"
"sort"
@ -403,6 +404,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
// Create new main with the same config.
config := cluster[2].Command.Config
config.Translation.MapSize = 100000
// config.Bind = cluster[2].API.Node().URI.HostPort()
// this isn't necessary, but makes the test run way faster
@ -476,6 +478,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
// Create new main with the same config.
config := cluster[2].Command.Config
config.Translation.MapSize = 100000
// config.Bind = cluster[2].API.Node().URI.HostPort()
// this isn't necessary, but makes the test run way faster
@ -496,6 +499,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
// Create new main with the same config.
config = cluster[1].Command.Config
// config.Bind = cluster[1].API.Node().URI.HostPort()
config.Translation.MapSize = 100000
// this isn't necessary, but makes the test run way faster
config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port))
@ -564,4 +568,128 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
}
}
// Ensure program imports timestamps as UTC.
func TestMain_ImportTimestamp(t *testing.T) {
m := test.MustRunCommand()
defer m.Close()
indexName := "i"
fieldName := "f"
// Create index.
if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
// Create field.
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil {
t.Fatal(err)
}
data := pilosa.ImportRequest{
Index: indexName,
Field: fieldName,
Shard: 0,
RowIDs: []uint64{1, 2},
ColumnIDs: []uint64{1, 2},
Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00
}
// Import data.
if err := m.API.Import(context.Background(), &data); err != nil {
t.Fatal(err)
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
exp := []string{
"standard", "standard_2018", "standard_201801", "standard_20180101",
"standard_2019", "standard_201912", "standard_20191231",
}
got := []string{}
for _, f := range files {
got = append(got, f.Name())
}
if !reflect.DeepEqual(got, exp) {
t.Fatalf("expected %v, but got %v", exp, got)
}
}
func TestClusterQueriesAfterRestart(t *testing.T) {
cluster := test.MustRunCluster(t, 3)
defer cluster.Close()
cmd1 := cluster[1]
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
// build a query to set the first bit in 100 shards
query := strings.Builder{}
for i := 0; i < 100; i++ {
query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth))
}
_, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: query.String(),
})
if err != nil {
t.Fatalf("setting 100 bits in 100 shards: %v", err)
}
results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: "Count(Row(testfield=0))",
})
if err != nil {
t.Fatalf("counting row: %v", err)
}
if results.Results[0].(uint64) != 100 {
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
}
err = cmd1.Command.Close()
if err != nil {
t.Fatalf("closing node0: %v", err)
}
// confirm that cluster stops accepting queries after one node closes
if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
}
// Create new main with the same config.
config := cmd1.Command.Config
config.Bind = cmd1.API.Node().URI.HostPort()
// this isn't necessary, but makes the test run way faster
config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port))
cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr)
cmd1.Command.Config = config
err = cmd1.Start()
if err != nil {
t.Fatalf("reopening node 0: %v", err)
}
for cmd1.API.State() != pilosa.ClusterStateNormal {
time.Sleep(time.Millisecond)
}
results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: "Count(Row(testfield=0))",
})
if err != nil {
t.Fatalf("counting row: %v", err)
}
if results.Results[0].(uint64) != 100 {
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
}
}
// TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.

View file

@ -55,16 +55,21 @@ func newCommand(opts ...server.CommandOption) *Command {
panic(err)
}
// set aggressive close timeout by default to avoid hanging tests. This was
// Set aggressive close timeout by default to avoid hanging tests. This was
// a problem with PDK tests which used go-pilosa as well. We put it at the
// beginning of the option slice so that it can be overridden by user-passed
// options.
opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...)
// Also set TranslateFile MapSize to a smaller number so memory allocation
// does not fail on 32-bit systems.
opts = append([]server.CommandOption{
server.OptCommandCloseTimeout(time.Millisecond * 2),
}, opts...)
m := &Command{commandOptions: opts}
m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Config.Translation.MapSize = 100000
if testing.Verbose() {
m.Command.Stdout = os.Stdout

View file

@ -5,7 +5,6 @@ import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
@ -17,6 +16,7 @@ import (
"time"
"github.com/cespare/xxhash"
"github.com/pkg/errors"
)
const (
@ -81,9 +81,29 @@ type TranslateFile struct {
replicationRetryInterval time.Duration
}
// TranslateFileOption is a functional option type for pilosa.TranslateFile
type TranslateFileOption func(f *TranslateFile) error
func OptTranslateFileMapSize(mapSize int) TranslateFileOption {
return func(f *TranslateFile) error {
f.mapSize = mapSize
return nil
}
}
// NewTranslateFile returns a new instance of TranslateFile.
func NewTranslateFile() *TranslateFile {
return &TranslateFile{
func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile {
var defaultMapSize64 int64 = 10 * (1 << 30)
var defaultMapSize int
if ^uint(0)>>32 > 0 {
// 10GB default map size
defaultMapSize = int(defaultMapSize64)
} else {
// Use 2GB default map size on 32-bit systems
defaultMapSize = (1 << 31) - 1
}
f := &TranslateFile{
writeNotify: make(chan struct{}),
closing: make(chan struct{}),
cols: make(map[string]*index),
@ -96,25 +116,35 @@ func NewTranslateFile() *TranslateFile {
replicationRetryInterval: defaultReplicationRetryInterval,
}
for _, opt := range opts {
err := opt(f)
if err != nil {
// TODO (2.0): Change func signature to return error
panic(errors.Wrap(err, "applying option"))
}
}
return f
}
func (s *TranslateFile) Open() (err error) {
// Open writer & buffered writer.
if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
return err
return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
} else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil {
return err
return errors.Wrapf(err, "open file %s", s.Path)
}
s.w = bufio.NewWriter(s.file)
// Memory map data file.
if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
return err
return errors.Wrapf(err, "creating Mmap (size: %d)", s.mapSize)
}
// Replay the log.
if err := s.replayEntries(); err != nil {
return err
return errors.Wrap(err, "replaying log entries")
}
// Listen to primaryStoreEvents channel.

View file

@ -1,5 +0,0 @@
package pilosa
// defaultMapSize is the default size of mapped memory for the translate store.
// It is passed as an int to syscall.Mmap and so must be < 2^31
const defaultMapSize = (1 << 31) - 1 // 2GB

View file

@ -1,8 +0,0 @@
// +build !386
package pilosa
// defaultMapSize is the default size of mapped memory for the translate store.
// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on
// 64bit systems.
const defaultMapSize = 10 * (1 << 30) // 10GB

View file

@ -367,7 +367,7 @@ func TestPrintTranslateFile(t *testing.T) {
}
f.Close()
s := pilosa.NewTranslateFile()
s := pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))
s.Path = f.Name()
err = s.Open()
if err != nil {
@ -809,7 +809,7 @@ func NewTranslateFile() *TranslateFile {
}
f.Close()
s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()}
s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))}
s.Path = f.Name()
return s
}
@ -847,7 +847,7 @@ func (s *TranslateFile) Reopen() error {
}
s.lock.Lock()
s.TranslateFile = pilosa.NewTranslateFile()
s.TranslateFile = pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))
s.lock.Unlock()
s.Path = prev.Path
s.SetPrimaryStore("restored-primary", prev.PrimaryTranslateStore)

View file

@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
if v.fieldType == FieldTypeMutex {
frag.mutexVector = newRowsVector(frag)
} else if v.fieldType == FieldTypeBool {
frag.mutexVector = newBoolVector(frag)
}
return frag
}