mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-07 11:27:50 +00:00
Merge branch 'master' into new-rows-iterate
This commit is contained in:
commit
36a539d24e
45 changed files with 3572 additions and 1638 deletions
|
|
@ -68,9 +68,20 @@ jobs:
|
|||
docker:
|
||||
- image: circleci/python:2.7-jessie
|
||||
steps:
|
||||
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
|
||||
- *fast-checkout
|
||||
- run: sudo pip install awscli
|
||||
- run: make prerelease-upload
|
||||
dockerhub-upload:
|
||||
<<: *defaults
|
||||
steps:
|
||||
- run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR
|
||||
- *fast-checkout
|
||||
- setup_remote_docker
|
||||
- run: make docker
|
||||
- run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master
|
||||
- run: docker login -u $DOCKER_USER -p $DOCKER_PASS
|
||||
- run: docker push pilosa/pilosa:master
|
||||
workflows:
|
||||
version: 2
|
||||
test:
|
||||
|
|
@ -104,3 +115,7 @@ workflows:
|
|||
- prerelease-upload:
|
||||
requires:
|
||||
- prerelease
|
||||
- dockerhub-upload:
|
||||
requires:
|
||||
- linter
|
||||
- test-golang-1.10
|
||||
|
|
|
|||
82
api.go
82
api.go
|
|
@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
|
|||
return QueryResponse{}, errors.Wrap(err, "validating api method")
|
||||
}
|
||||
|
||||
resp := QueryResponse{}
|
||||
|
||||
q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
|
||||
if err != nil {
|
||||
return resp, errors.Wrap(err, "parsing")
|
||||
return QueryResponse{}, errors.Wrap(err, "parsing")
|
||||
}
|
||||
execOpts := &execOptions{
|
||||
Remote: req.Remote,
|
||||
|
|
@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
|
|||
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
|
||||
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
|
||||
}
|
||||
results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
|
||||
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
|
||||
if err != nil {
|
||||
return resp, errors.Wrap(err, "executing")
|
||||
return QueryResponse{}, errors.Wrap(err, "executing")
|
||||
}
|
||||
resp.Results = results
|
||||
|
||||
// Fill column attributes if requested.
|
||||
// execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true)
|
||||
if execOpts.ColumnAttrs {
|
||||
// Consolidate all column ids across all calls.
|
||||
var columnIDs []uint64
|
||||
for _, result := range results {
|
||||
bm, ok := result.(*Row)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
|
||||
}
|
||||
|
||||
// Retrieve column attributes across all calls.
|
||||
columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs)
|
||||
if err != nil {
|
||||
return resp, errors.Wrap(err, "reading column attrs")
|
||||
}
|
||||
|
||||
// Translate column attributes, if necessary.
|
||||
if api.holder.translateFile != nil {
|
||||
for _, col := range resp.ColumnAttrSets {
|
||||
v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
col.Key, col.ID = v, 0
|
||||
}
|
||||
}
|
||||
|
||||
resp.ColumnAttrSets = columnAttrSets
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// readColumnAttrSets returns a list of column attribute objects by id.
|
||||
func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
|
||||
if index == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ax := make([]*ColumnAttrSet, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
// Read attributes for column. Skip column if empty.
|
||||
attrs, err := index.ColumnAttrStore().Attrs(id)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting attrs")
|
||||
} else if len(attrs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Append column with attributes.
|
||||
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
|
||||
}
|
||||
|
||||
return ax, nil
|
||||
}
|
||||
|
||||
// CreateIndex makes a new Pilosa index.
|
||||
func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) {
|
||||
if err := api.validate(apiCreateIndex); err != nil {
|
||||
|
|
@ -321,13 +263,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
|
|||
nodes := api.cluster.shardNodes(indexName, shard)
|
||||
var eg errgroup.Group
|
||||
|
||||
field := api.holder.Field(indexName, fieldName)
|
||||
if field == nil {
|
||||
return newNotFoundError(ErrFieldNotFound)
|
||||
}
|
||||
|
||||
// only set fields are supported
|
||||
if field.Type() != FieldTypeSet {
|
||||
return NewBadRequestError(errors.New("roaring import is only supported for set fields"))
|
||||
}
|
||||
|
||||
for _, node := range nodes {
|
||||
node := node
|
||||
if node.ID == api.server.nodeID {
|
||||
field := api.holder.Field(indexName, fieldName)
|
||||
if field == nil {
|
||||
return newNotFoundError(ErrFieldNotFound)
|
||||
}
|
||||
// must make a copy of data to operate on locally. field.importRoaring changes data
|
||||
d2 := make([]byte, len(data))
|
||||
copy(d2, data)
|
||||
|
|
@ -732,7 +680,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error {
|
|||
if ts == 0 {
|
||||
continue
|
||||
}
|
||||
t := time.Unix(0, ts)
|
||||
t := time.Unix(0, ts).UTC()
|
||||
timestamps[i] = &t
|
||||
}
|
||||
|
||||
|
|
@ -801,6 +749,8 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error {
|
|||
}
|
||||
|
||||
// MaxShards returns the maximum shard number for each index in a map.
|
||||
// TODO (2.0): This method has been deprecated. Instead, use
|
||||
// AvailableShardsByIndex.
|
||||
func (api *API) MaxShards(_ context.Context) map[string]uint64 {
|
||||
m := make(map[string]uint64)
|
||||
for k, v := range api.holder.availableShardsByIndex() {
|
||||
|
|
|
|||
6
cache.go
6
cache.go
|
|
@ -167,8 +167,10 @@ func NewRankCache(maxEntries uint32) *rankCache {
|
|||
func (c *rankCache) Add(id uint64, n uint64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
// Ignore if the column count is below the threshold.
|
||||
if n < c.thresholdValue {
|
||||
// Ignore if the column count is below the threshold,
|
||||
// unless the count is 0, which is effectively used
|
||||
// to clear the cache value.
|
||||
if n < c.thresholdValue && n > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) {
|
|||
tests := []commandTest{
|
||||
// TEST 0
|
||||
{
|
||||
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"},
|
||||
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"},
|
||||
env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
|
||||
cfgFileContent: `
|
||||
data-dir = "/tmp/myFileDatadir"
|
||||
|
|
@ -65,13 +65,14 @@ func TestServerConfig(t *testing.T) {
|
|||
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"})
|
||||
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
|
||||
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
|
||||
v.Check(cmd.Server.Config.Translation.MapSize, 100000)
|
||||
return v.Error()
|
||||
},
|
||||
},
|
||||
// TEST 1
|
||||
{
|
||||
args: []string{"server", "--anti-entropy.interval", "9m0s"},
|
||||
env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110"},
|
||||
env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"},
|
||||
cfgFileContent: `
|
||||
bind = "localhost:0"
|
||||
data-dir = "` + actualDataDir + `"
|
||||
|
|
@ -85,12 +86,13 @@ func TestServerConfig(t *testing.T) {
|
|||
v := validator{}
|
||||
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
|
||||
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
|
||||
v.Check(cmd.Server.Config.Translation.MapSize, 100000)
|
||||
return v.Error()
|
||||
},
|
||||
},
|
||||
// TEST 2
|
||||
{
|
||||
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"},
|
||||
args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"},
|
||||
env: map[string]string{},
|
||||
cfgFileContent: `
|
||||
bind = "localhost:19444"
|
||||
|
|
|
|||
|
|
@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
|
|||
t.Fatalf("Import Run with values doesn't work: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that import into bool field runs.
|
||||
func TestImportCommand_RunBool(t *testing.T) {
|
||||
buf := bytes.Buffer{}
|
||||
stdin, stdout, stderr := GetIO(buf)
|
||||
cm := NewImportCommand(stdin, stdout, stderr)
|
||||
ctx := context.Background()
|
||||
|
||||
cmd := test.MustRunCluster(t, 1)[0]
|
||||
cm.Host = cmd.API.Node().URI.HostPort()
|
||||
|
||||
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
|
||||
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`)))
|
||||
|
||||
cm.Index = "i"
|
||||
cm.Field = "f"
|
||||
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
file, err := ioutil.TempFile("", "import-bool.csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file.Write([]byte("0,1\n1,2\n1,3"))
|
||||
|
||||
cm.Paths = []string{file.Name()}
|
||||
err = cm.Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Import Run to bool field doesn't work: %s", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Ensure that invalid bool values return an error.
|
||||
t.Run("Invalid", func(t *testing.T) {
|
||||
file, err := ioutil.TempFile("", "import-invalid-bool.csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file.Write([]byte("0,1\n1,2\n1,3\n2,4"))
|
||||
|
||||
cm.Paths = []string{file.Name()}
|
||||
err = cm.Run(ctx)
|
||||
if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") {
|
||||
t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
|
|||
|
||||
// Translation
|
||||
flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
|
||||
flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
|
||||
|
||||
// Gossip
|
||||
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
|
||||
|
|
|
|||
|
|
@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// EnrichWithCPUInfo adds CPU information to the diagnostics payload.
|
||||
func (d *diagnosticsCollector) EnrichWithCPUInfo() {
|
||||
d.Set("CPUArch", d.server.systemInfo.CPUArch())
|
||||
}
|
||||
|
||||
// EnrichWithOSInfo adds OS information to the diagnostics payload.
|
||||
func (d *diagnosticsCollector) EnrichWithOSInfo() {
|
||||
uptime, err := d.server.systemInfo.Uptime()
|
||||
|
|
@ -265,6 +270,7 @@ type SystemInfo interface {
|
|||
MemFree() (uint64, error)
|
||||
MemTotal() (uint64, error)
|
||||
MemUsed() (uint64, error)
|
||||
CPUArch() string
|
||||
}
|
||||
|
||||
// newNopSystemInfo creates a no-op implementation of SystemInfo.
|
||||
|
|
@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) {
|
|||
func (n *nopSystemInfo) MemUsed() (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// CPUArch returns the CPU architecture, such as amd64
|
||||
func (n *nopSystemInfo) CPUArch() string {
|
||||
return ""
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the
|
|||
pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
|
||||
```
|
||||
|
||||
##### Importing Boolean Values
|
||||
|
||||
If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true).
|
||||
|
||||
For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row.
|
||||
```
|
||||
0,3
|
||||
0,9
|
||||
1,1
|
||||
1,2
|
||||
1,4
|
||||
1,8
|
||||
```
|
||||
|
||||
<div class="note">
|
||||
<p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option
|
|||
* `int`
|
||||
* `min` (int): Minimum integer value allowed for the field.
|
||||
* `max` (int): Maximum integer value allowed for the field.
|
||||
* `bool`
|
||||
* (boolean fields take no arguments)
|
||||
* `time`
|
||||
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
|
||||
* `mutex`
|
||||
|
|
|
|||
|
|
@ -307,6 +307,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
|
|||
skip-verify = true
|
||||
```
|
||||
|
||||
#### Translation Map Size
|
||||
|
||||
* Description: Size in bytes of mmap to allocate for key translation
|
||||
* Flag: `translation.map-size`
|
||||
* Env: `PILOSA_TRANSLATION_MAP_SIZE`
|
||||
* Config:
|
||||
|
||||
```toml
|
||||
[translation]
|
||||
map-size = 10737418240
|
||||
```
|
||||
|
||||
### Example Cluster Configuration
|
||||
|
||||
A three node cluster running on different hosts could be minimally configured as follows:
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust
|
|||
|
||||
### Field Type
|
||||
|
||||
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`.
|
||||
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`.
|
||||
|
||||
#### Set
|
||||
|
||||
|
|
@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00)
|
|||
#### Mutex
|
||||
|
||||
Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field.
|
||||
|
||||
#### Boolean
|
||||
|
||||
A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ nav = []
|
|||
|
||||
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
|
||||
|
||||
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
|
||||
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
|
||||
|
||||
<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.
|
||||
|
||||
|
|
|
|||
|
|
@ -236,6 +236,70 @@ Clear(10, stargazer=1)
|
|||
|
||||
This represents removing the relationship between the user with id=1 and the repository with id=10.
|
||||
|
||||
#### ClearRow
|
||||
|
||||
**Spec:**
|
||||
|
||||
```
|
||||
ClearRow(<FIELD>=<ROW>)
|
||||
```
|
||||
|
||||
**Description:**
|
||||
|
||||
`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns.
|
||||
|
||||
**Result Type:** boolean
|
||||
|
||||
A return value of `true` indicates that at least one column was toggled from 1 to 0.
|
||||
|
||||
A return value of `false` indicates that all bits in the row were already 0 and nothing changed.
|
||||
|
||||
**Examples:**
|
||||
|
||||
Clear all bit in row 1 in the stargazer field:
|
||||
```request
|
||||
ClearRow(stargazer=1)
|
||||
```
|
||||
```response
|
||||
{"results":[true]}
|
||||
```
|
||||
|
||||
This represents removing the relationship between the user with id=1 and all repositories.
|
||||
|
||||
#### Store
|
||||
|
||||
**Spec:**
|
||||
|
||||
```
|
||||
Store(<ROW_CALL>, <FIELD>=<ROW>)
|
||||
```
|
||||
|
||||
**Description:**
|
||||
|
||||
`Store` writes the results of <ROW_CALL> to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`.
|
||||
|
||||
**Result Type:** boolean
|
||||
|
||||
Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call.
|
||||
|
||||
**Examples:**
|
||||
|
||||
Store the contents of stargazer row 1 into stargazer row 2:
|
||||
```request
|
||||
Store(Row(stargazer=1), stargazer=2)
|
||||
```
|
||||
```response
|
||||
{"results":[true]}
|
||||
```
|
||||
|
||||
Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20.
|
||||
```request
|
||||
Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20)
|
||||
```
|
||||
```response
|
||||
{"results":[true]}
|
||||
```
|
||||
|
||||
### Read Operations
|
||||
|
||||
#### Row
|
||||
|
|
|
|||
|
|
@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \
|
|||
{"success":true}
|
||||
```
|
||||
|
||||
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
|
||||
Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
|
||||
|
||||
The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
|
||||
``` request
|
||||
|
|
|
|||
|
|
@ -1,3 +1,17 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package proto
|
||||
|
||||
import (
|
||||
|
|
@ -812,30 +826,32 @@ func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent)
|
|||
|
||||
func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
|
||||
m.Node = &pilosa.Node{}
|
||||
decodeIndexStatuses(pb.Indexes, m.Indexes)
|
||||
m.Indexes = decodeIndexStatuses(pb.Indexes)
|
||||
m.Schema = &pilosa.Schema{}
|
||||
decodeSchema(pb.Schema, m.Schema)
|
||||
}
|
||||
|
||||
func decodeIndexStatuses(a []*internal.IndexStatus, m []*pilosa.IndexStatus) {
|
||||
m = m[:0]
|
||||
func decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.IndexStatus {
|
||||
m := make([]*pilosa.IndexStatus, 0)
|
||||
for i := range a {
|
||||
m = append(m, &pilosa.IndexStatus{})
|
||||
decodeIndexStatus(a[i], m[i])
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) {
|
||||
m.Name = pb.Name
|
||||
decodeFieldStatuses(pb.Fields, m.Fields)
|
||||
m.Fields = decodeFieldStatuses(pb.Fields)
|
||||
}
|
||||
|
||||
func decodeFieldStatuses(a []*internal.FieldStatus, m []*pilosa.FieldStatus) {
|
||||
m = m[:0]
|
||||
func decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.FieldStatus {
|
||||
m := make([]*pilosa.FieldStatus, 0)
|
||||
for i := range a {
|
||||
m = append(m, &pilosa.FieldStatus{})
|
||||
decodeFieldStatus(a[i], m[i])
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) {
|
||||
|
|
|
|||
|
|
@ -88,13 +88,13 @@ func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Containe
|
|||
// this struct is added to prevent the closure locals from being escaped out to the heap
|
||||
type updater struct {
|
||||
key uint64
|
||||
n int
|
||||
n int32
|
||||
containerType byte
|
||||
mapped bool
|
||||
}
|
||||
|
||||
func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
|
||||
a := updater{key, n, containerType, mapped}
|
||||
a := updater{key, int32(n), containerType, mapped}
|
||||
btc.tree.Put(key, a.update)
|
||||
}
|
||||
|
||||
|
|
|
|||
329
executor.go
329
executor.go
|
|
@ -81,20 +81,21 @@ func newExecutor(opts ...executorOption) *executor {
|
|||
}
|
||||
|
||||
// Execute executes a PQL query.
|
||||
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
|
||||
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
|
||||
resp := QueryResponse{}
|
||||
// Verify that an index is set.
|
||||
if index == "" {
|
||||
return nil, ErrIndexRequired
|
||||
return resp, ErrIndexRequired
|
||||
}
|
||||
|
||||
idx := e.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return nil, ErrIndexNotFound
|
||||
return resp, ErrIndexNotFound
|
||||
}
|
||||
|
||||
// Verify that the number of writes do not exceed the maximum.
|
||||
if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest {
|
||||
return nil, ErrTooManyWrites
|
||||
return resp, ErrTooManyWrites
|
||||
}
|
||||
|
||||
// Default options.
|
||||
|
|
@ -107,14 +108,48 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
|
|||
if !opt.Remote {
|
||||
for i := range q.Calls {
|
||||
if err := e.translateCall(index, idx, q.Calls[i]); err != nil {
|
||||
return nil, err
|
||||
return resp, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results, err := e.execute(ctx, index, q, shards, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return resp, err
|
||||
}
|
||||
|
||||
resp.Results = results
|
||||
|
||||
// Fill column attributes if requested.
|
||||
if opt.ColumnAttrs {
|
||||
// Consolidate all column ids across all calls.
|
||||
var columnIDs []uint64
|
||||
for _, result := range results {
|
||||
bm, ok := result.(*Row)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
|
||||
}
|
||||
|
||||
// Retrieve column attributes across all calls.
|
||||
columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs)
|
||||
if err != nil {
|
||||
return resp, errors.Wrap(err, "reading column attrs")
|
||||
}
|
||||
|
||||
// Translate column attributes, if necessary.
|
||||
if idx.Keys() {
|
||||
for _, col := range columnAttrSets {
|
||||
v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
col.Key, col.ID = v, 0
|
||||
}
|
||||
}
|
||||
|
||||
resp.ColumnAttrSets = columnAttrSets
|
||||
}
|
||||
|
||||
// Translate response objects from ids to keys, if necessary.
|
||||
|
|
@ -123,11 +158,35 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
|
|||
for i := range results {
|
||||
results[i], err = e.translateResult(index, idx, q.Calls[i], results[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return resp, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// readColumnAttrSets returns a list of column attribute objects by id.
|
||||
func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
|
||||
if index == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ax := make([]*ColumnAttrSet, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
// Read attributes for column. Skip column if empty.
|
||||
attrs, err := index.ColumnAttrStore().Attrs(id)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting attrs")
|
||||
} else if len(attrs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Append column with attributes.
|
||||
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
|
||||
}
|
||||
|
||||
return ax, nil
|
||||
}
|
||||
|
||||
func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) {
|
||||
|
|
@ -184,6 +243,10 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
|
|||
return e.executeMax(ctx, index, c, shards, opt)
|
||||
case "Clear":
|
||||
return e.executeClearBit(ctx, index, c, opt)
|
||||
case "ClearRow":
|
||||
return e.executeClearRow(ctx, index, c, shards, opt)
|
||||
case "Store":
|
||||
return e.executeSetRow(ctx, index, c, shards, opt)
|
||||
case "Count":
|
||||
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
|
||||
return e.executeCount(ctx, index, c, shards, opt)
|
||||
|
|
@ -1055,8 +1118,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca
|
|||
rowID, rowOK, rowErr := c.UintArg(fieldName)
|
||||
if rowErr != nil {
|
||||
return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr)
|
||||
}
|
||||
if !rowOK {
|
||||
} else if !rowOK {
|
||||
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
|
||||
}
|
||||
|
||||
|
|
@ -1429,7 +1491,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
|
|||
return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
|
||||
}
|
||||
|
||||
// executeClearBitField executes a Clear() call for a single view.
|
||||
// executeClearBitField executes a Clear() call for a field.
|
||||
func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) {
|
||||
shard := colID / ShardWidth
|
||||
ret := false
|
||||
|
|
@ -1459,9 +1521,170 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// executeClearRow executes a ClearRow() call.
|
||||
func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
|
||||
// Ensure the field type supports ClearRow().
|
||||
fieldName, err := c.FieldArg()
|
||||
if err != nil {
|
||||
return false, errors.New("ClearRow() argument required: field")
|
||||
}
|
||||
field := e.Holder.Field(index, fieldName)
|
||||
if field == nil {
|
||||
return false, ErrFieldNotFound
|
||||
}
|
||||
|
||||
switch field.Type() {
|
||||
case FieldTypeSet, FieldTypeTime, FieldTypeMutex, FieldTypeBool:
|
||||
// These field types support ClearRow().
|
||||
default:
|
||||
return false, fmt.Errorf("ClearRow() is not supported on %s field types", field.Type())
|
||||
}
|
||||
|
||||
// Execute calls in bulk on each remote node and merge.
|
||||
mapFn := func(shard uint64) (interface{}, error) {
|
||||
return e.executeClearRowShard(ctx, index, c, shard)
|
||||
}
|
||||
|
||||
// Merge returned results at coordinating node.
|
||||
reduceFn := func(prev, v interface{}) interface{} {
|
||||
val := v.(bool)
|
||||
if prev == nil {
|
||||
return val
|
||||
}
|
||||
return val || prev.(bool)
|
||||
}
|
||||
|
||||
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
|
||||
return result.(bool), err
|
||||
}
|
||||
|
||||
// executeClearRowShard executes a ClearRow() call for a single shard.
|
||||
func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
|
||||
fieldName, err := c.FieldArg()
|
||||
if err != nil {
|
||||
return false, errors.New("ClearRow() argument required: field")
|
||||
}
|
||||
|
||||
// Read fields using labels.
|
||||
rowID, ok, err := c.UintArg(fieldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading ClearRow() row: %v", err)
|
||||
} else if !ok {
|
||||
return false, fmt.Errorf("ClearRow() row argument '%v' required", rowLabel)
|
||||
}
|
||||
|
||||
field := e.Holder.Field(index, fieldName)
|
||||
if field == nil {
|
||||
return false, ErrFieldNotFound
|
||||
}
|
||||
|
||||
// Remove the row from all views.
|
||||
changed := false
|
||||
for _, view := range field.views() {
|
||||
fragment := e.Holder.fragment(index, fieldName, view.name, shard)
|
||||
if fragment == nil {
|
||||
continue
|
||||
}
|
||||
cleared, err := fragment.clearRow(rowID)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard)
|
||||
}
|
||||
changed = changed || cleared
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// executeSetRow executes a SetRow() call.
|
||||
func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
|
||||
// Ensure the field type supports SetRow().
|
||||
fieldName, err := c.FieldArg()
|
||||
if err != nil {
|
||||
return false, errors.New("SetRow() argument required: field")
|
||||
}
|
||||
field := e.Holder.Field(index, fieldName)
|
||||
if field == nil {
|
||||
return false, ErrFieldNotFound
|
||||
}
|
||||
if field.Type() != FieldTypeSet {
|
||||
return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type())
|
||||
}
|
||||
|
||||
// Execute calls in bulk on each remote node and merge.
|
||||
mapFn := func(shard uint64) (interface{}, error) {
|
||||
return e.executeSetRowShard(ctx, index, c, shard)
|
||||
}
|
||||
|
||||
// Merge returned results at coordinating node.
|
||||
reduceFn := func(prev, v interface{}) interface{} {
|
||||
val := v.(bool)
|
||||
if prev == nil {
|
||||
return val
|
||||
}
|
||||
return val || prev.(bool)
|
||||
}
|
||||
|
||||
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
|
||||
return result.(bool), err
|
||||
}
|
||||
|
||||
// executeSetRowShard executes a SetRow() call for a single shard.
|
||||
func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
|
||||
fieldName, err := c.FieldArg()
|
||||
if err != nil {
|
||||
return false, errors.New("SetRow() argument required: field")
|
||||
}
|
||||
|
||||
// Read fields using labels.
|
||||
rowID, ok, err := c.UintArg(fieldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading SetRow() row: %v", err)
|
||||
} else if !ok {
|
||||
return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel)
|
||||
}
|
||||
|
||||
field := e.Holder.Field(index, fieldName)
|
||||
if field == nil {
|
||||
return false, ErrFieldNotFound
|
||||
}
|
||||
|
||||
// Retrieve source row.
|
||||
var src *Row
|
||||
if len(c.Children) == 1 {
|
||||
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "getting source row")
|
||||
}
|
||||
src = row
|
||||
} else {
|
||||
return false, errors.New("SetRow() requires a source row")
|
||||
}
|
||||
|
||||
// Set the row on the standard view.
|
||||
changed := false
|
||||
fragment := e.Holder.fragment(index, fieldName, viewStandard, shard)
|
||||
if fragment == nil {
|
||||
// Since the destination fragment doesn't exist, create one.
|
||||
view, err := field.createViewIfNotExists(viewStandard)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "creating view")
|
||||
}
|
||||
fragment, err = view.createFragmentIfNotExists(shard)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "creating fragment: %d", shard)
|
||||
}
|
||||
}
|
||||
set, err := fragment.setRow(src, rowID)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard)
|
||||
}
|
||||
changed = changed || set
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// executeSet executes a Set() call.
|
||||
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
|
||||
|
||||
// Read colID.
|
||||
colID, ok, err := c.UintArg("_" + columnLabel)
|
||||
if err != nil {
|
||||
|
|
@ -1493,8 +1716,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
|
|||
}
|
||||
}
|
||||
|
||||
// Int field.
|
||||
if f.Type() == FieldTypeInt {
|
||||
// Read remaining fields using labels.
|
||||
// Read row value.
|
||||
rowVal, ok, err := c.IntArg(fieldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading Set() row: %v", err)
|
||||
|
|
@ -1503,27 +1727,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
|
|||
}
|
||||
|
||||
return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
|
||||
} else {
|
||||
// Read remaining fields using labels.
|
||||
rowID, ok, err := c.UintArg(fieldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading Set() row: %v", err)
|
||||
} else if !ok {
|
||||
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
|
||||
}
|
||||
|
||||
var timestamp *time.Time
|
||||
sTimestamp, ok := c.Args["_timestamp"].(string)
|
||||
if ok {
|
||||
t, err := time.Parse(TimeFormat, sTimestamp)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("invalid date: %s", sTimestamp)
|
||||
}
|
||||
timestamp = &t
|
||||
}
|
||||
|
||||
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
|
||||
}
|
||||
|
||||
// Read row ID.
|
||||
rowID, ok, err := c.UintArg(fieldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading Set() row: %v", err)
|
||||
} else if !ok {
|
||||
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
|
||||
}
|
||||
|
||||
var timestamp *time.Time
|
||||
sTimestamp, ok := c.Args["_timestamp"].(string)
|
||||
if ok {
|
||||
t, err := time.Parse(TimeFormat, sTimestamp)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("invalid date: %s", sTimestamp)
|
||||
}
|
||||
timestamp = &t
|
||||
}
|
||||
|
||||
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
|
||||
}
|
||||
|
||||
// executeSetBitField executes a Set() call for a specific field.
|
||||
|
|
@ -1954,21 +2178,22 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
|
|||
|
||||
func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
|
||||
var colKey, rowKey, fieldName string
|
||||
if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" {
|
||||
switch c.Name {
|
||||
case "Set", "Clear", "Row", "Range", "SetColumnAttrs":
|
||||
// Positional args in new PQL syntax require special handling here.
|
||||
colKey = "_" + columnLabel
|
||||
fieldName, _ = c.FieldArg()
|
||||
rowKey = fieldName
|
||||
} else if c.Name == "SetRowAttrs" {
|
||||
case "SetRowAttrs":
|
||||
// Positional args in new PQL syntax require special handling here.
|
||||
rowKey = "_" + rowLabel
|
||||
fieldName = callArgString(c, "_field")
|
||||
} else if c.Name == "Rows" {
|
||||
case "Rows":
|
||||
fieldName = callArgString(c, "field")
|
||||
rowKey = "previous"
|
||||
} else if c.Name == "GroupBy" {
|
||||
case "GroupBy":
|
||||
return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy")
|
||||
} else {
|
||||
default:
|
||||
colKey = "col"
|
||||
fieldName = callArgString(c, "field")
|
||||
rowKey = "row"
|
||||
|
|
@ -2001,7 +2226,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
|
|||
// will raise an error downstream when it's used.
|
||||
return nil
|
||||
}
|
||||
if field.keys() {
|
||||
|
||||
// Bool field keys do not use the translator because there
|
||||
// are only two possible values. Instead, they are handled
|
||||
// directly.
|
||||
if field.Type() == FieldTypeBool {
|
||||
boolVal, err := callArgBool(c, rowKey)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting bool key")
|
||||
}
|
||||
rowID := falseRowID
|
||||
if boolVal {
|
||||
rowID = trueRowID
|
||||
}
|
||||
c.Args[rowKey] = rowID
|
||||
} else if field.keys() {
|
||||
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
|
||||
return errors.New("row value must be a string when field 'keys' option enabled")
|
||||
}
|
||||
|
|
@ -2268,6 +2507,18 @@ func (vc *ValCount) larger(other ValCount) ValCount {
|
|||
}
|
||||
}
|
||||
|
||||
func callArgBool(call *pql.Call, key string) (bool, error) {
|
||||
value, ok := call.Args[key]
|
||||
if !ok {
|
||||
return false, errors.New("missing bool argument")
|
||||
}
|
||||
b, ok := value.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("invalid bool argument type: %T", value)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func callArgString(call *pql.Call, key string) string {
|
||||
value, ok := call.Args[key]
|
||||
if !ok {
|
||||
|
|
|
|||
551
executor_test.go
551
executor_test.go
|
|
@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// Ensure a set query can be executed on a bool field.
|
||||
func TestExecutor_Execute_SetBool(t *testing.T) {
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
|
||||
// Create fields.
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set a true bit.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !res.Results[0].(bool) {
|
||||
t.Fatalf("expected column changed")
|
||||
}
|
||||
|
||||
// Set the same bit to true again verify nothing changed.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res.Results[0].(bool) {
|
||||
t.Fatalf("expected column to be unchanged")
|
||||
}
|
||||
|
||||
// Set the same bit to false.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !res.Results[0].(bool) {
|
||||
t.Fatalf("expected column changed")
|
||||
}
|
||||
|
||||
// Ensure that the false row is set.
|
||||
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) {
|
||||
t.Fatalf("unexpected colums: %+v", columns)
|
||||
}
|
||||
|
||||
// Ensure that the true row is empty.
|
||||
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
|
||||
t.Fatalf("unexpected colums: %+v", columns)
|
||||
}
|
||||
})
|
||||
t.Run("Error", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
|
||||
// Create fields.
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bool using a string value.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil {
|
||||
t.Fatalf("expected invalid bool type error")
|
||||
}
|
||||
|
||||
// Set bool using an integer.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil {
|
||||
t.Fatalf("expected invalid bool type error")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure old PQL syntax doesn't break anything too badly.
|
||||
func TestExecutor_Execute_OldPQL(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
|
|
@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
|
|||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
|
||||
// Create felds.
|
||||
// Create fields.
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -948,6 +1020,58 @@ func TestExecutor_Execute_Range(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// Ensure a range query with keys can be executed.
|
||||
func TestExecutor_Execute_Range_WithKeys(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
|
||||
// Create index.
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
|
||||
// Create field.
|
||||
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), pilosa.OptFieldKeys()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set columns.
|
||||
cc := `
|
||||
Set(2, f="foo", 1999-12-31T00:00)
|
||||
Set(3, f="foo", 2000-01-01T00:00)
|
||||
Set(4, f="foo", 2000-01-02T00:00)
|
||||
Set(5, f="foo", 2000-02-01T00:00)
|
||||
Set(6, f="foo", 2001-01-01T00:00)
|
||||
Set(7, f="foo", 2002-01-01T02:00)
|
||||
|
||||
Set(2, f="foo", 1999-12-30T00:00)
|
||||
Set(2, f="foo", 2002-02-01T00:00)
|
||||
Set(2, f="bar", 2001-01-01T00:00)
|
||||
`
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("Standard", func(t *testing.T) {
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Clear", func(t *testing.T) {
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f="foo")`}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure a Range(bsiGroup) query can be executed.
|
||||
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
|
|
@ -1453,6 +1577,36 @@ func TestExecutor_QueryCall(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("columnAttrsWithKeys", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
|
||||
// Set columns for rows 0, 10, & 20 across two shards.
|
||||
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
|
||||
Set("one-hundred", f="ten")
|
||||
SetColumnAttrs("one-hundred", foo="bar")
|
||||
`}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
targetColAttrSets := []*pilosa.ColumnAttrSet{
|
||||
{Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}},
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f="ten"), columnAttrs=true)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) {
|
||||
t.Fatalf("unexpected keys: %+v", keys)
|
||||
} else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) {
|
||||
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("shards", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
|
|
@ -1601,6 +1755,401 @@ func TestExecutor_Execute_Not(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a row can be cleared.
|
||||
func TestExecutor_Execute_ClearRow(t *testing.T) {
|
||||
t.Run("Set", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Clear the row and ensure we get a `true` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Clear the row again and ensure we get a `false` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row is empty.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Ensure other rows were not affected.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
})
|
||||
t.Run("Mutex", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Clear the row and ensure we get a `true` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Clear the row again and ensure we get a `false` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row is empty.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Ensure other rows were not affected.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
})
|
||||
t.Run("Time", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set columns.
|
||||
cc := `
|
||||
Set(2, f=1, 1999-12-31T00:00)
|
||||
Set(3, f=1, 2000-01-01T00:00)
|
||||
Set(4, f=1, 2000-01-02T00:00)
|
||||
Set(5, f=1, 2000-02-01T00:00)
|
||||
Set(6, f=1, 2001-01-01T00:00)
|
||||
Set(7, f=1, 2002-01-01T02:00)
|
||||
|
||||
Set(2, f=1, 1999-12-30T00:00)
|
||||
Set(2, f=1, 2002-02-01T00:00)
|
||||
Set(2, f=10, 2001-01-01T00:00)
|
||||
`
|
||||
rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`
|
||||
rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`
|
||||
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
|
||||
// Clear the row and ensure we get a `true` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row is empty.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
|
||||
// Ensure other rows were not affected.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
})
|
||||
t.Run("Int", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ensure that clearing a row raises an error.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil {
|
||||
t.Fatal("expected clear row to return an error")
|
||||
}
|
||||
})
|
||||
t.Run("TopN", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cc := `
|
||||
Set(2, f=1)
|
||||
Set(3, f=1)
|
||||
Set(4, f=1)
|
||||
Set(5, f=1)
|
||||
Set(6, f=1)
|
||||
Set(7, f=1)
|
||||
Set(8, f=1)
|
||||
|
||||
Set(2, f=2)
|
||||
Set(3, f=2)
|
||||
Set(4, f=2)
|
||||
Set(5, f=2)
|
||||
Set(6, f=2)
|
||||
Set(7, f=2)
|
||||
|
||||
Set(2, f=3)
|
||||
Set(3, f=3)
|
||||
Set(4, f=3)
|
||||
Set(5, f=3)
|
||||
Set(6, f=3)
|
||||
`
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := c[0].RecalculateCaches(); err != nil {
|
||||
t.Fatalf("recalculating caches: %v", err)
|
||||
}
|
||||
|
||||
// Check the TopN results.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
|
||||
{ID: 1, Count: 7},
|
||||
{ID: 2, Count: 6},
|
||||
{ID: 3, Count: 5},
|
||||
}}) {
|
||||
t.Fatalf("topn wrong results: %v", res.Results)
|
||||
}
|
||||
|
||||
// Clear the row and ensure we get a `true` response.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected clear row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache).
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
|
||||
{ID: 1, Count: 7},
|
||||
{ID: 3, Count: 5},
|
||||
}}) {
|
||||
t.Fatalf("topn wrong results: %v", res.Results)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure a row can be set.
|
||||
func TestExecutor_Execute_SetRow(t *testing.T) {
|
||||
t.Run("Set_NewRow", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := index.CreateField("tmp", pilosa.OptFieldTypeDefault()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Store row 10 into a different row.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected set row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row was populated.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
})
|
||||
t.Run("Set_NoSource", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Store row 9 (which doesn't exist) into a different row.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected set row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row was populated.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Store row 9 (which doesn't exist) into a row that does exist.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected set row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row was populated.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
})
|
||||
t.Run("Set_ExistingDestination", func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := test.Holder{Holder: c[0].Server.Holder()}
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
|
||||
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits.
|
||||
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
|
||||
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
|
||||
// Store row 10 into an existing row.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res := res.Results[0].(bool); !res {
|
||||
t.Fatalf("unexpected set row result: %+v", res)
|
||||
}
|
||||
|
||||
// Ensure the row was populated.
|
||||
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
|
||||
t.Fatalf("unexpected columns: %+v", bits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkExistence(nn bool, b *testing.B) {
|
||||
c := test.MustRunCluster(b, 1)
|
||||
defer c.Close()
|
||||
|
|
|
|||
95
field.go
95
field.go
|
|
@ -52,6 +52,7 @@ const (
|
|||
FieldTypeInt = "int"
|
||||
FieldTypeTime = "time"
|
||||
FieldTypeMutex = "mutex"
|
||||
FieldTypeBool = "bool"
|
||||
)
|
||||
|
||||
// Field represents a container for views.
|
||||
|
|
@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption {
|
|||
}
|
||||
}
|
||||
|
||||
func OptFieldTypeBool() FieldOption {
|
||||
return func(fo *FieldOptions) error {
|
||||
if fo.Type != "" {
|
||||
return errors.Errorf("field type is already set to: %s", fo.Type)
|
||||
}
|
||||
fo.Type = FieldTypeBool
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewField returns a new instance of field.
|
||||
func NewField(path, index, name string, opts FieldOption) (*Field, error) {
|
||||
err := validateName(name)
|
||||
|
|
@ -219,8 +230,17 @@ func (f *Field) AvailableShards() *roaring.Bitmap {
|
|||
return b
|
||||
}
|
||||
|
||||
// addRemoteAvailableShards merges the set of available shards into the current known set.
|
||||
func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) {
|
||||
// addRemoteAvailableShards merges the set of available shards into the current known set
|
||||
// and saves the set to a file.
|
||||
func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error {
|
||||
f.mergeRemoteAvailableShards(b)
|
||||
|
||||
// Save the updated bitmap to the data store.
|
||||
return f.saveAvailableShards()
|
||||
}
|
||||
|
||||
// mergeRemoteAvailableShards merges the set of available shards into the current known set.
|
||||
func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.remoteAvailableShards = f.remoteAvailableShards.Union(b)
|
||||
|
|
@ -280,6 +300,10 @@ func (f *Field) Open() error {
|
|||
return errors.Wrap(err, "loading meta")
|
||||
}
|
||||
|
||||
if err := f.loadAvailableShards(); err != nil {
|
||||
return errors.Wrap(err, "loading available shards")
|
||||
}
|
||||
|
||||
// Apply the field options loaded from meta.
|
||||
if err := f.applyOptions(f.options); err != nil {
|
||||
return errors.Wrap(err, "applying options")
|
||||
|
|
@ -441,6 +465,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
|
|||
f.options.Max = 0
|
||||
f.options.TimeQuantum = ""
|
||||
f.options.Keys = opt.Keys
|
||||
case FieldTypeBool:
|
||||
f.options.Type = FieldTypeBool
|
||||
f.options.CacheType = CacheTypeNone
|
||||
f.options.CacheSize = 0
|
||||
f.options.Min = 0
|
||||
f.options.Max = 0
|
||||
f.options.TimeQuantum = ""
|
||||
f.options.Keys = false
|
||||
default:
|
||||
return errors.New("invalid field type")
|
||||
}
|
||||
|
|
@ -448,6 +480,47 @@ func (f *Field) applyOptions(opt FieldOptions) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// loadAvailableShards reads remoteAvailableShards data for the field, if any.
|
||||
func (f *Field) loadAvailableShards() error {
|
||||
bm := roaring.NewBitmap()
|
||||
|
||||
// Read data from meta file.
|
||||
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards"))
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return errors.Wrap(err, "reading available shards")
|
||||
} else {
|
||||
if err := bm.UnmarshalBinary(buf); err != nil {
|
||||
return errors.Wrap(err, "unmarshaling")
|
||||
}
|
||||
}
|
||||
|
||||
// Merge bitmap from file into field.
|
||||
f.mergeRemoteAvailableShards(bm)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveAvailableShards writes remoteAvailableShards data for the field.
|
||||
func (f *Field) saveAvailableShards() error {
|
||||
// Open or create file.
|
||||
file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "opening available shards file")
|
||||
}
|
||||
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
|
||||
// Write available shards to file.
|
||||
if _, err := f.remoteAvailableShards.WriteTo(file); err != nil {
|
||||
return errors.Wrap(err, "writing bitmap to buffer")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the field and its views.
|
||||
func (f *Field) Close() error {
|
||||
f.mu.Lock()
|
||||
|
|
@ -681,6 +754,10 @@ func (f *Field) deleteView(name string) error {
|
|||
}
|
||||
|
||||
// Row returns a row of the standard view.
|
||||
// It seems this method is only being used by the test
|
||||
// package, and the fact that it's only allowed on
|
||||
// `set` fields is odd. This may be considered for
|
||||
// deprecation in a future version.
|
||||
func (f *Field) Row(rowID uint64) (*Row, error) {
|
||||
if f.Type() != FieldTypeSet {
|
||||
return nil, errors.Errorf("row method unsupported for field type: %s", f.Type())
|
||||
|
|
@ -954,10 +1031,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
|
|||
return errors.New("time quantum not set in field")
|
||||
}
|
||||
|
||||
fieldType := f.Type()
|
||||
|
||||
// Split import data by fragment.
|
||||
dataByFragment := make(map[importKey]importData)
|
||||
for i := range rowIDs {
|
||||
rowID, columnID := rowIDs[i], columnIDs[i]
|
||||
|
||||
// Bool-specific data validation.
|
||||
if fieldType == FieldTypeBool && rowID > 1 {
|
||||
return errors.New("bool field imports only support values 0 and 1")
|
||||
}
|
||||
|
||||
var timestamp *time.Time
|
||||
if len(timestamps) > i {
|
||||
timestamp = timestamps[i]
|
||||
|
|
@ -1191,6 +1276,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
|
|||
o.CacheSize,
|
||||
o.Keys,
|
||||
})
|
||||
case FieldTypeBool:
|
||||
return json.Marshal(struct {
|
||||
Type string `json:"type"`
|
||||
}{
|
||||
o.Type,
|
||||
})
|
||||
}
|
||||
return nil, errors.New("invalid field type")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/pilosa/pilosa/pql"
|
||||
"github.com/pilosa/pilosa/roaring"
|
||||
)
|
||||
|
||||
// Ensure a bsiGroup can adjust to its baseValue.
|
||||
|
|
@ -341,3 +342,22 @@ func TestField_RowTime(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
func TestField_PersistAvailableShards(t *testing.T) {
|
||||
f := MustOpenField(OptFieldTypeDefault())
|
||||
|
||||
// bm represents remote available shards.
|
||||
bm := roaring.NewBitmap(1, 2, 3)
|
||||
|
||||
if err := f.addRemoteAvailableShards(bm); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reload field and verify that shard data is persisted.
|
||||
if err := f.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
|
||||
t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
272
fragment.go
272
fragment.go
|
|
@ -75,6 +75,10 @@ const (
|
|||
|
||||
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
|
||||
defaultFragmentMaxOpN = 2000
|
||||
|
||||
// Row ids used for boolean fields.
|
||||
falseRowID = uint64(0)
|
||||
trueRowID = uint64(1)
|
||||
)
|
||||
|
||||
// fragment represents the intersection of a field and shard in an index.
|
||||
|
|
@ -351,11 +355,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row {
|
|||
}
|
||||
|
||||
// Only use a subset of the containers.
|
||||
// NOTE: The start & end ranges must be divisible by
|
||||
// NOTE: The start & end ranges must be divisible by container width.
|
||||
data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
|
||||
|
||||
// Reference bitmap subrange in storage.
|
||||
// We Clone() data because otherwise row will contains pointers to containers in storage.
|
||||
// We Clone() data because otherwise row will contain pointers to containers in storage.
|
||||
// This causes unexpected results when we cache the row and try to use it later.
|
||||
row := &Row{
|
||||
segments: []rowSegment{{
|
||||
|
|
@ -390,12 +394,13 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
|
|||
// handleMutex will clear an existing row and store the new row
|
||||
// in the vector.
|
||||
func (f *fragment) handleMutex(rowID, columnID uint64) error {
|
||||
if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID {
|
||||
if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
|
||||
return errors.Wrap(err, "getting mutex vector data")
|
||||
} else if found && existingRowID != rowID {
|
||||
if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil {
|
||||
return errors.Wrap(err, "clearing mutex value")
|
||||
}
|
||||
}
|
||||
f.mutexVector.Set(columnID, rowID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -489,6 +494,95 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
|
|||
return changed, nil
|
||||
}
|
||||
|
||||
// setRow replaces an existing row (specified by rowID) with the given
|
||||
// Row. This updates both the on-disk storage and the in-cache bitmap.
|
||||
func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.unprotectedSetRow(row, rowID)
|
||||
}
|
||||
|
||||
func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) {
|
||||
// TODO: In order to return `changed`, we need to first compare
|
||||
// the existing row with the given row. Determine if the overhead
|
||||
// of this is worth having `changed`.
|
||||
// For now we will assume changed is always true.
|
||||
changed = true
|
||||
|
||||
// First container of the row in storage.
|
||||
headContainerKey := rowID << shardVsContainerExponent
|
||||
|
||||
// Remove every existing container in the row.
|
||||
for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
|
||||
f.storage.Containers.Remove(headContainerKey + i)
|
||||
}
|
||||
|
||||
// From the given row, get the rowSegment for this shard.
|
||||
seg := row.segment(f.shard)
|
||||
if seg == nil {
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// Put each container from rowSegment to fragment storage.
|
||||
citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent)
|
||||
for citer.Next() {
|
||||
k, c := citer.Value()
|
||||
f.storage.Containers.Put(headContainerKey+(k%(1<<shardVsContainerExponent)), c)
|
||||
}
|
||||
|
||||
// Update the row in cache.
|
||||
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
|
||||
f.cache.BulkAdd(rowID, n)
|
||||
|
||||
// Snapshot storage.
|
||||
if err := f.snapshot(); err != nil {
|
||||
return false, errors.Wrap(err, "snapshotting")
|
||||
}
|
||||
|
||||
f.stats.Count("setRow", 1, 1.0)
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// ClearRow clears a row for a given rowID within the fragment.
|
||||
// This updates both the on-disk storage and the in-cache bitmap.
|
||||
func (f *fragment) clearRow(rowID uint64) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.unprotectedClearRow(rowID)
|
||||
}
|
||||
|
||||
func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
|
||||
changed = false
|
||||
|
||||
// First container of the row in storage.
|
||||
headContainerKey := rowID << shardVsContainerExponent
|
||||
|
||||
// Remove every container in the row.
|
||||
for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
|
||||
k := headContainerKey + i
|
||||
// Technically we could bypass the Get() call and only
|
||||
// call Remove(), but the Get() gives us the ability
|
||||
// to return true if any existing data was removed.
|
||||
if cont := f.storage.Containers.Get(k); cont != nil {
|
||||
f.storage.Containers.Remove(k)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
// Clear the row in cache.
|
||||
f.cache.Add(rowID, 0)
|
||||
|
||||
// Snapshot storage.
|
||||
if err := f.snapshot(); err != nil {
|
||||
return false, errors.Wrap(err, "snapshotting")
|
||||
}
|
||||
|
||||
f.stats.Count("clearRow", 1, 1.0)
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (f *fragment) bit(rowID, columnID uint64) (bool, error) {
|
||||
pos, err := f.pos(rowID, columnID)
|
||||
if err != nil {
|
||||
|
|
@ -1333,16 +1427,29 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
|
|||
return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs))
|
||||
}
|
||||
|
||||
if f.mutexVector != nil {
|
||||
return f.bulkImportMutex(rowIDs, columnIDs)
|
||||
}
|
||||
return f.bulkImportStandard(rowIDs, columnIDs)
|
||||
}
|
||||
|
||||
// bulkImportStandard performs a bulk import on a standard fragment.
|
||||
func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64) error {
|
||||
// Create a temporary bitmap which will be populated by rowIDs and columnIDs
|
||||
// and then merged into the existing fragment's bitmap.
|
||||
localBitmap := roaring.NewBitmap()
|
||||
|
||||
// Disconnect op writer so we don't append updates.
|
||||
localBitmap.OpWriter = nil
|
||||
|
||||
// Process every bit.
|
||||
// If an error occurs then reopen the storage.
|
||||
lastID := uint64(0)
|
||||
// rowSet maintains the set of rowIDs present in this import.
|
||||
// It allows the cache to be updated once per row, instead of once
|
||||
// per bit.
|
||||
rowSet := make(map[uint64]struct{})
|
||||
lastRowID := uint64(0)
|
||||
|
||||
// Process every bit by writing to a local bitmap,
|
||||
// to be merged with fragment storage next.
|
||||
for i := range rowIDs {
|
||||
rowID, columnID := rowIDs[i], columnIDs[i]
|
||||
|
||||
|
|
@ -1352,18 +1459,18 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Write to storage.
|
||||
// Write to local storage.
|
||||
_, err = localBitmap.Add(pos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Reduce the StatsD rate for high volume stats
|
||||
f.stats.Count("ImportBit", 1, 0.0001)
|
||||
// import optimization to avoid linear foreach calls
|
||||
// slight risk of concurrent cache counter being off but
|
||||
// no real danger
|
||||
if i == 0 || rowID != lastID {
|
||||
lastID = rowID
|
||||
|
||||
// Add row to rowSet.
|
||||
if i == 0 || rowID != lastRowID {
|
||||
lastRowID = rowID
|
||||
rowSet[rowID] = struct{}{}
|
||||
}
|
||||
|
||||
|
|
@ -1392,6 +1499,98 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error {
|
|||
return unprotectedWriteToFragment(f, results)
|
||||
}
|
||||
|
||||
// bulkImportMutex performs a bulk import on a fragment while ensuring
|
||||
// mutex restrictions. Because the mutex requirements must be checked
|
||||
// against storage, this method must acquire a write lock on the fragment
|
||||
// during the entire process, and it handles every bit independently.
|
||||
func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
// Disconnect op writer so we don't append updates.
|
||||
f.storage.OpWriter = nil
|
||||
|
||||
// If an error occurs then reopen the storage.
|
||||
if err := func() error {
|
||||
// rowSet maintains the set of rowIDs present in this import.
|
||||
// It allows the cache to be updated once per row, instead of once
|
||||
// per bit.
|
||||
rowSet := make(map[uint64]struct{})
|
||||
lastRowID := uint64(0)
|
||||
|
||||
// Process every bit.
|
||||
for i := range rowIDs {
|
||||
rowID, columnID := rowIDs[i], columnIDs[i]
|
||||
|
||||
// Handle mutex vector (i.e. clear an existing row).
|
||||
if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
|
||||
return errors.Wrap(err, "getting mutex vector data")
|
||||
} else if found && existingRowID != rowID {
|
||||
// Determine the position of the bit in the storage.
|
||||
pos, err := f.pos(existingRowID, columnID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clear storage.
|
||||
_, err = f.storage.Remove(pos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rowSet[existingRowID] = struct{}{}
|
||||
}
|
||||
|
||||
// Determine the position of the bit in the storage.
|
||||
pos, err := f.pos(rowID, columnID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write to storage.
|
||||
_, err = f.storage.Add(pos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Reduce the StatsD rate for high volume stats
|
||||
f.stats.Count("ImportBit", 1, 0.0001)
|
||||
|
||||
// Add row to rowSet.
|
||||
if i == 0 || rowID != lastRowID {
|
||||
lastRowID = rowID
|
||||
rowSet[rowID] = struct{}{}
|
||||
}
|
||||
|
||||
// Invalidate block checksum.
|
||||
delete(f.checksums, int(rowID/HashBlockSize))
|
||||
}
|
||||
|
||||
// Update cache counts for all rows.
|
||||
for rowID := range rowSet {
|
||||
// Import should ALWAYS have row() load a new bm from fragment.storage
|
||||
// because the row that's in rowCache hasn't been updated with
|
||||
// this import's data.
|
||||
f.cache.BulkAdd(rowID, f.unprotectedRow(rowID).Count())
|
||||
}
|
||||
|
||||
f.cache.Invalidate()
|
||||
|
||||
return nil
|
||||
}(); err != nil {
|
||||
_ = f.closeStorage()
|
||||
_ = f.openStorage()
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the storage to disk and reload.
|
||||
if err := f.snapshot(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importValue bulk imports a set of range-encoded values.
|
||||
func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error {
|
||||
f.mu.Lock()
|
||||
|
|
@ -2073,8 +2272,7 @@ func pos(rowID, columnID uint64) uint64 {
|
|||
// vector stores the mapping of colID to rowID.
|
||||
// It's used for a mutex field type.
|
||||
type vector interface {
|
||||
Get(colID uint64) (uint64, bool)
|
||||
Set(colID, rowID uint64)
|
||||
Get(colID uint64) (uint64, bool, error)
|
||||
}
|
||||
|
||||
// rowsVector implements the vector interface by looking
|
||||
|
|
@ -2093,20 +2291,50 @@ func newRowsVector(f *fragment) *rowsVector {
|
|||
// Get returns the rowID associated to the given colID.
|
||||
// Additionally, it returns true if a value was found,
|
||||
// otherwise it returns false.
|
||||
func (v *rowsVector) Get(colID uint64) (uint64, bool) {
|
||||
func (v *rowsVector) Get(colID uint64) (uint64, bool, error) {
|
||||
rows := v.f.rows(0, filterColumn(colID))
|
||||
if len(rows) == 1 {
|
||||
return rows[0], true
|
||||
if len(rows) > 1 {
|
||||
return 0, false, errors.New("found multiple row values for column")
|
||||
} else if len(rows) == 1 {
|
||||
return rows[0], true, nil
|
||||
}
|
||||
return 0, false
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
// Set is not used for rowsVector.
|
||||
func (v *rowsVector) Set(colID, rowID uint64) {}
|
||||
|
||||
// rowToKey converts a Pilosa row ID to the key of the container which starts
|
||||
// that row in the bitmap which represents this entire fragment. A fragment is
|
||||
// all the rows within a shard within a field concatenated together.
|
||||
func rowToKey(rowID uint64) (key uint64) {
|
||||
return rowID * (ShardWidth / containerWidth)
|
||||
}
|
||||
|
||||
// boolVector implements the vector interface by looking
|
||||
// at data in rows 0 and 1.
|
||||
type boolVector struct {
|
||||
f *fragment
|
||||
}
|
||||
|
||||
// newBoolVector returns a boolVector for a given fragment.
|
||||
func newBoolVector(f *fragment) *boolVector {
|
||||
return &boolVector{
|
||||
f: f,
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns the rowID associated to the given colID.
|
||||
// Additionally, it returns true if a value was found,
|
||||
// otherwise it returns false.
|
||||
func (v *boolVector) Get(colID uint64) (uint64, bool, error) {
|
||||
rows := v.f.rows(0, filterColumn(colID))
|
||||
if len(rows) > 1 {
|
||||
return 0, false, errors.New("found multiple row values for column")
|
||||
} else if len(rows) == 1 {
|
||||
switch rows[0] {
|
||||
case falseRowID, trueRowID:
|
||||
return rows[0], true, nil
|
||||
default:
|
||||
return 0, false, errors.New("found non-boolean value")
|
||||
}
|
||||
}
|
||||
return 0, false, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,6 +95,81 @@ func TestFragment_ClearBit(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can clear a row.
|
||||
func TestFragment_ClearRow(t *testing.T) {
|
||||
f := mustOpenFragment("i", "f", viewStandard, 0, "")
|
||||
defer f.Close()
|
||||
|
||||
// Set and then clear bits on the fragment.
|
||||
if _, err := f.setBit(1000, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.setBit(1000, 65536); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.unprotectedClearRow(1000); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify count on row.
|
||||
if n := f.row(1000).Count(); n != 0 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.row(1000).Count(); n != 0 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can set a row.
|
||||
func TestFragment_SetRow(t *testing.T) {
|
||||
f := mustOpenFragment("i", "f", viewStandard, 7, "")
|
||||
defer f.Close()
|
||||
|
||||
rowID := uint64(1000)
|
||||
|
||||
// Set bits on the fragment.
|
||||
if _, err := f.setBit(rowID, 8000001); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.setBit(rowID, 8065536); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data on row.
|
||||
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
|
||||
t.Fatalf("unexpected columns: %+v", cols)
|
||||
}
|
||||
// Verify count on row.
|
||||
if n := f.row(rowID).Count(); n != 2 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
// Set row (overwrite existing data).
|
||||
row := NewRow(8000002, 8065537, 8131074)
|
||||
if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !changed {
|
||||
t.Fatalf("expected changed value: %v", changed)
|
||||
}
|
||||
|
||||
// Verify data on row.
|
||||
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
|
||||
t.Fatalf("unexpected columns after set row: %+v", cols)
|
||||
}
|
||||
// Verify count on row.
|
||||
if n := f.row(rowID).Count(); n != 3 {
|
||||
t.Fatalf("unexpected count after set row: %d", n)
|
||||
}
|
||||
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.row(rowID).Count(); n != 3 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can set & read a value.
|
||||
func TestFragment_SetValue(t *testing.T) {
|
||||
t.Run("OK", func(t *testing.T) {
|
||||
|
|
@ -1178,6 +1253,146 @@ func TestFragment_SetMutex(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can import mutually exclusive values.
|
||||
func TestFragment_ImportMutex(t *testing.T) {
|
||||
tests := []struct {
|
||||
rowIDs []uint64
|
||||
colIDs []uint64
|
||||
exp map[uint64][]uint64
|
||||
}{
|
||||
{
|
||||
[]uint64{1, 1, 1, 1},
|
||||
[]uint64{0, 1, 2, 3},
|
||||
map[uint64][]uint64{
|
||||
1: {0, 1, 2, 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{1, 1, 1, 1, 2, 2, 2, 2},
|
||||
[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
|
||||
map[uint64][]uint64{
|
||||
1: {},
|
||||
2: {0, 1, 2, 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{1, 1, 1, 1, 2},
|
||||
[]uint64{0, 1, 2, 3, 1},
|
||||
map[uint64][]uint64{
|
||||
1: {0, 2, 3},
|
||||
2: {1},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{1, 1, 1, 1, 2, 2, 1},
|
||||
[]uint64{0, 1, 2, 3, 1, 8, 1},
|
||||
map[uint64][]uint64{
|
||||
1: {0, 1, 2, 3},
|
||||
2: {8},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{1, 2, 3},
|
||||
[]uint64{8, 8, 8},
|
||||
map[uint64][]uint64{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {8},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
|
||||
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
|
||||
defer f.Close()
|
||||
|
||||
err := f.bulkImport(test.rowIDs, test.colIDs)
|
||||
if err != nil {
|
||||
t.Fatalf("bulk importing ids: %v", err)
|
||||
}
|
||||
|
||||
// Check for expected results.
|
||||
for k, v := range test.exp {
|
||||
cols := f.row(k).Columns()
|
||||
if !reflect.DeepEqual(cols, v) {
|
||||
t.Fatalf("expected: %v, but got: %v", v, cols)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can import bool values.
|
||||
func TestFragment_ImportBool(t *testing.T) {
|
||||
tests := []struct {
|
||||
rowIDs []uint64
|
||||
colIDs []uint64
|
||||
exp map[uint64][]uint64
|
||||
}{
|
||||
{
|
||||
[]uint64{1, 1, 1, 1},
|
||||
[]uint64{0, 1, 2, 3},
|
||||
map[uint64][]uint64{
|
||||
1: {0, 1, 2, 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{0, 0, 0, 0, 1, 1, 1, 1},
|
||||
[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
|
||||
map[uint64][]uint64{
|
||||
0: {},
|
||||
1: {0, 1, 2, 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{0, 0, 0, 0, 1},
|
||||
[]uint64{0, 1, 2, 3, 1},
|
||||
map[uint64][]uint64{
|
||||
0: {0, 2, 3},
|
||||
1: {1},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{1, 1, 1, 1, 0, 0, 1},
|
||||
[]uint64{0, 1, 2, 3, 1, 8, 1},
|
||||
map[uint64][]uint64{
|
||||
0: {8},
|
||||
1: {0, 1, 2, 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
[]uint64{0, 1, 2},
|
||||
[]uint64{8, 8, 8},
|
||||
map[uint64][]uint64{
|
||||
0: {},
|
||||
1: {}, // This isn't {8} because fragment doesn't validate bool values.
|
||||
2: {8},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
|
||||
f := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
|
||||
defer f.Close()
|
||||
|
||||
err := f.bulkImport(test.rowIDs, test.colIDs)
|
||||
if err != nil {
|
||||
t.Fatalf("bulk importing ids: %v", err)
|
||||
}
|
||||
|
||||
// Check for expected results.
|
||||
for k, v := range test.exp {
|
||||
cols := f.row(k).Columns()
|
||||
if !reflect.DeepEqual(cols, v) {
|
||||
t.Fatalf("expected: %v, but got: %v", v, cols)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragment_Snapshot(b *testing.B) {
|
||||
if *FragmentPath == "" {
|
||||
b.Skip("no fragment specified")
|
||||
|
|
@ -1302,6 +1517,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st
|
|||
return frag
|
||||
}
|
||||
|
||||
// mustOpenBoolFragment returns a new instance of Fragment for a bool field.
|
||||
func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment {
|
||||
frag := mustOpenFragment(index, field, view, shard, cacheType)
|
||||
frag.mutexVector = newBoolVector(frag)
|
||||
return frag
|
||||
}
|
||||
|
||||
// Reopen closes the fragment and reopens it as a new instance.
|
||||
func (f *fragment) reopen() error {
|
||||
if err := f.Close(); err != nil {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@
|
|||
package gopsutil
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/shirou/gopsutil/host"
|
||||
"github.com/shirou/gopsutil/mem"
|
||||
|
|
@ -109,6 +111,11 @@ func (s *systemInfo) KernelVersion() (string, error) {
|
|||
return host.KernelVersion()
|
||||
}
|
||||
|
||||
// CPUArch returns the CPU architecture, such as amd64
|
||||
func (s *systemInfo) CPUArch() string {
|
||||
return runtime.GOARCH
|
||||
}
|
||||
|
||||
// NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo.
|
||||
func NewSystemInfo() *systemInfo {
|
||||
return &systemInfo{}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
package gopsutil_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
|
|
@ -25,15 +24,6 @@ import (
|
|||
func TestSystemInfo(t *testing.T) {
|
||||
var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo()
|
||||
|
||||
// Uptime()(uint64, error)
|
||||
// Platform()(string, error)
|
||||
// Family()(string, error)
|
||||
// OSVersion()(string, error)
|
||||
// KernelVersion()(string, error)
|
||||
// MemFree()(uint64, error)
|
||||
// MemTotal()(uint64, error)
|
||||
// MemUsed()(uint64, error)
|
||||
//
|
||||
uptime, err := systemInfo.Uptime()
|
||||
if err != nil || uptime == 0 {
|
||||
t.Fatalf("Error collecting uptime (error: %v)", err)
|
||||
|
|
@ -70,8 +60,12 @@ func TestSystemInfo(t *testing.T) {
|
|||
}
|
||||
|
||||
memtotal, err := systemInfo.MemTotal()
|
||||
log.Println(memtotal)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err)
|
||||
}
|
||||
|
||||
cpuArch := systemInfo.CPUArch()
|
||||
if cpuArch == "" {
|
||||
t.Fatalf("Error getting CPU arch.")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
114
http/handler.go
114
http/handler.go
|
|
@ -170,13 +170,34 @@ func (h *Handler) Close() error {
|
|||
|
||||
func (h *Handler) populateValidators() {
|
||||
h.validators = map[string]*queryValidationSpec{}
|
||||
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
|
||||
h.validators["GetShardMax"] = queryValidationSpecRequired()
|
||||
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
|
||||
h.validators["Home"] = queryValidationSpecRequired()
|
||||
h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired()
|
||||
h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired()
|
||||
h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired()
|
||||
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard")
|
||||
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
|
||||
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard")
|
||||
h.validators["GetIndexes"] = queryValidationSpecRequired()
|
||||
h.validators["GetIndex"] = queryValidationSpecRequired()
|
||||
h.validators["PostIndex"] = queryValidationSpecRequired()
|
||||
h.validators["DeleteIndex"] = queryValidationSpecRequired()
|
||||
h.validators["PostField"] = queryValidationSpecRequired()
|
||||
h.validators["DeleteField"] = queryValidationSpecRequired()
|
||||
h.validators["PostImport"] = queryValidationSpecRequired()
|
||||
h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote")
|
||||
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns")
|
||||
h.validators["GetInfo"] = queryValidationSpecRequired()
|
||||
h.validators["RecalculateCaches"] = queryValidationSpecRequired()
|
||||
h.validators["GetSchema"] = queryValidationSpecRequired()
|
||||
h.validators["GetStatus"] = queryValidationSpecRequired()
|
||||
h.validators["GetVersion"] = queryValidationSpecRequired()
|
||||
h.validators["PostClusterMessage"] = queryValidationSpecRequired()
|
||||
h.validators["GetFragmentBlockData"] = queryValidationSpecRequired()
|
||||
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
|
||||
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
|
||||
h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired()
|
||||
h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
|
||||
h.validators["GetNodes"] = queryValidationSpecRequired()
|
||||
h.validators["GetShardMax"] = queryValidationSpecRequired()
|
||||
h.validators["GetTranslateData"] = queryValidationSpecRequired("offset")
|
||||
}
|
||||
|
||||
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
|
||||
|
|
@ -203,40 +224,40 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
|
|||
// newRouter creates a new mux http router.
|
||||
func newRouter(handler *Handler) *mux.Router {
|
||||
router := mux.NewRouter()
|
||||
router.HandleFunc("/", handler.handleHome).Methods("GET")
|
||||
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
|
||||
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
|
||||
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
|
||||
router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home")
|
||||
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
|
||||
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
|
||||
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator")
|
||||
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
|
||||
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
|
||||
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
|
||||
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
|
||||
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET")
|
||||
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
|
||||
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
|
||||
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes")
|
||||
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex")
|
||||
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
|
||||
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
|
||||
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
|
||||
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
|
||||
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST")
|
||||
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
|
||||
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField")
|
||||
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport")
|
||||
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring")
|
||||
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
|
||||
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
|
||||
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
|
||||
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
|
||||
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
|
||||
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
|
||||
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo")
|
||||
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches")
|
||||
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema")
|
||||
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus")
|
||||
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
|
||||
|
||||
// /internal endpoints are for internal use only; they may change at any time.
|
||||
// DO NOT rely on these for external applications!
|
||||
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST")
|
||||
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
|
||||
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
|
||||
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData")
|
||||
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
|
||||
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
|
||||
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
|
||||
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
|
||||
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
|
||||
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
|
||||
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
|
||||
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
|
||||
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET")
|
||||
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
|
||||
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
|
||||
|
||||
// TODO: Apply MethodNotAllowed statuses to all endpoints.
|
||||
// Ideally this would be automatic, as described in this (wontfix) ticket:
|
||||
|
|
@ -521,7 +542,12 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
|
|||
return err
|
||||
}
|
||||
// Unmarshal expected values.
|
||||
var _p _postIndexRequest
|
||||
_p := _postIndexRequest{
|
||||
Options: pilosa.IndexOptions{
|
||||
Keys: false,
|
||||
TrackExistence: true,
|
||||
},
|
||||
}
|
||||
if err := json.Unmarshal(b, &_p); err != nil {
|
||||
return errors.Wrap(err, "unmarshalling expected values")
|
||||
}
|
||||
|
|
@ -597,7 +623,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
|
|||
resp := successResponse{}
|
||||
|
||||
// Decode request.
|
||||
var req postIndexRequest
|
||||
req := postIndexRequest{
|
||||
Options: pilosa.IndexOptions{
|
||||
Keys: false,
|
||||
TrackExistence: true,
|
||||
},
|
||||
}
|
||||
err := json.NewDecoder(r.Body).Decode(&req)
|
||||
if err != nil && err != io.EOF {
|
||||
resp.write(w, err)
|
||||
|
|
@ -687,6 +718,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
|
|||
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
|
||||
case pilosa.FieldTypeMutex:
|
||||
fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize))
|
||||
case pilosa.FieldTypeBool:
|
||||
fos = append(fos, pilosa.OptFieldTypeBool())
|
||||
}
|
||||
if req.Options.Keys != nil {
|
||||
if *req.Options.Keys {
|
||||
|
|
@ -778,6 +811,20 @@ func (o *fieldOptions) validate() error {
|
|||
} else if o.TimeQuantum != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
|
||||
}
|
||||
case pilosa.FieldTypeBool:
|
||||
if o.CacheType != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool"))
|
||||
} else if o.CacheSize != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool"))
|
||||
} else if o.Min != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool"))
|
||||
} else if o.Max != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool"))
|
||||
} else if o.TimeQuantum != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
|
||||
} else if o.Keys != nil {
|
||||
return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("invalid field type: %s", o.Type)
|
||||
}
|
||||
|
|
@ -1457,11 +1504,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
resp := &pilosa.ImportResponse{}
|
||||
// TODO give meaningful stats for import
|
||||
err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body)
|
||||
resp := &pilosa.ImportResponse{}
|
||||
if err != nil {
|
||||
resp.Err = err.Error()
|
||||
if _, ok := err.(pilosa.BadRequestError); ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
// Marshal response object.
|
||||
buf, err := h.api.Serializer.Marshal(resp)
|
||||
|
|
|
|||
|
|
@ -31,8 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
|
|||
expected postIndexRequest
|
||||
err string
|
||||
}{
|
||||
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
|
||||
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}},
|
||||
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}},
|
||||
{json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}},
|
||||
{json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}},
|
||||
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
|
||||
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
|
||||
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
|
||||
|
|
@ -53,7 +54,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
|
|||
|
||||
if test.err == "" {
|
||||
if !reflect.DeepEqual(*actual, test.expected) {
|
||||
t.Errorf("expected: %v, but got: %v", test.expected, *actual)
|
||||
t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
7
index.go
7
index.go
|
|
@ -69,9 +69,10 @@ func NewIndex(path, name string) (*Index, error) {
|
|||
newAttrStore: newNopAttrStore,
|
||||
columnAttrs: nopStore,
|
||||
|
||||
broadcaster: NopBroadcaster,
|
||||
Stats: NopStatsClient,
|
||||
logger: NopLogger,
|
||||
broadcaster: NopBroadcaster,
|
||||
Stats: NopStatsClient,
|
||||
logger: NopLogger,
|
||||
trackExistence: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
|
|||
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
|
||||
// Can have a set of attributes attached to it.
|
||||
type ColumnAttrSet struct {
|
||||
ID uint64 `json:"id"`
|
||||
ID uint64 `json:"id,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Attrs map[string]interface{} `json:"attrs,omitempty"`
|
||||
}
|
||||
|
|
|
|||
17
pql/ast.go
17
pql/ast.go
|
|
@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) {
|
|||
return "", fmt.Errorf("No field argument specified")
|
||||
}
|
||||
|
||||
// BoolArg is for reading the value at key from call.Args as a bool. If the
|
||||
// key is not in Call.Args, the value of the returned bool will be false, and
|
||||
// the error will be nil. The value is assumed to be a bool. An error is
|
||||
// returned if the value is not a bool.
|
||||
func (c *Call) BoolArg(key string) (bool, bool, error) {
|
||||
val, ok := c.Args[key]
|
||||
if !ok {
|
||||
return false, false, nil
|
||||
}
|
||||
switch tval := val.(type) {
|
||||
case bool:
|
||||
return tval, true, nil
|
||||
default:
|
||||
return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval)
|
||||
}
|
||||
}
|
||||
|
||||
// UintArg is for reading the value at key from call.Args as a uint64. If the
|
||||
// key is not in Call.Args, the value of the returned bool will be false, and
|
||||
// the error will be nil. The value is assumed to be a uint64 or an int64 and
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
|
|||
/ 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
|
||||
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
|
||||
/ 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
|
||||
/ 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
|
||||
/ 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
|
||||
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
|
||||
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
|
||||
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
|
||||
|
|
|
|||
2592
pql/pql.peg.go
2592
pql/pql.peg.go
File diff suppressed because it is too large
Load diff
|
|
@ -51,13 +51,13 @@ func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n
|
|||
if i < 0 {
|
||||
c := NewContainer()
|
||||
c.containerType = containerType
|
||||
c.n = n
|
||||
c.n = int32(n)
|
||||
c.mapped = mapped
|
||||
sc.insertAt(key, c, -i-1)
|
||||
} else {
|
||||
c := sc.containers[i]
|
||||
c.containerType = containerType
|
||||
c.n = n
|
||||
c.n = int32(n)
|
||||
c.mapped = mapped
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ const (
|
|||
// at the beginning of every serialized run container.
|
||||
runCountHeaderSize = 2
|
||||
|
||||
// interval32Size is the size of a single run in a container.runs.
|
||||
// interval16Size is the size of a single run in a container.runs.
|
||||
interval16Size = 4
|
||||
|
||||
// bitmapN is the number of values in a container.bitmap.
|
||||
|
|
@ -73,6 +73,7 @@ type Containers interface {
|
|||
|
||||
// PutContainerValues updates an existing container at key.
|
||||
// If a container does not exist for key, a new one is allocated.
|
||||
// TODO(2.0) make n int32
|
||||
PutContainerValues(key uint64, containerType byte, n int, mapped bool)
|
||||
|
||||
// Remove takes the container at key out.
|
||||
|
|
@ -242,7 +243,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
|
|||
if found && skey == ekey {
|
||||
citer.Next()
|
||||
_, c := citer.Value()
|
||||
return uint64(c.countRange(int(lowbits(start)), int(lowbits(end))))
|
||||
return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end))))
|
||||
}
|
||||
|
||||
for citer.Next() {
|
||||
|
|
@ -252,7 +253,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
|
|||
panic("should be impossible for k to be less than skey")
|
||||
}
|
||||
if k == skey {
|
||||
n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1))
|
||||
n += uint64(c.countRange(int32(lowbits(start)), maxContainerVal+1))
|
||||
continue
|
||||
}
|
||||
if k < ekey {
|
||||
|
|
@ -260,7 +261,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
|
|||
continue
|
||||
}
|
||||
if k == ekey {
|
||||
n += uint64(c.countRange(0, int(lowbits(end))))
|
||||
n += uint64(c.countRange(0, int32(lowbits(end))))
|
||||
break
|
||||
}
|
||||
if k > ekey {
|
||||
|
|
@ -805,7 +806,7 @@ type Iterator struct {
|
|||
citer ContainerIterator
|
||||
key uint64
|
||||
c *Container
|
||||
j, k int // i: container; j: array index, bit index, or run index; k: offset within the run
|
||||
j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run
|
||||
}
|
||||
|
||||
// Seek moves to the first value equal to or greater than `seek`.
|
||||
|
|
@ -830,7 +831,7 @@ func (itr *Iterator) Seek(seek uint64) {
|
|||
if itr.j < 0 {
|
||||
itr.j = -itr.j - 1
|
||||
}
|
||||
if itr.j < len(itr.c.array) {
|
||||
if itr.j < int32(len(itr.c.array)) {
|
||||
itr.j--
|
||||
return
|
||||
}
|
||||
|
|
@ -853,7 +854,7 @@ func (itr *Iterator) Seek(seek uint64) {
|
|||
j, contains := binSearchRuns(lb, itr.c.runs)
|
||||
if contains {
|
||||
itr.j = j
|
||||
itr.k = int(lb) - int(itr.c.runs[j].start) - 1
|
||||
itr.k = int32(lb) - int32(itr.c.runs[j].start) - 1
|
||||
} else {
|
||||
// Set iterator to next value in the Bitmap.
|
||||
itr.j = j
|
||||
|
|
@ -864,7 +865,7 @@ func (itr *Iterator) Seek(seek uint64) {
|
|||
}
|
||||
|
||||
// If it's a bitmap container then move to index before the value and call next().
|
||||
itr.j = int(lb) - 1
|
||||
itr.j = int32(lb) - 1
|
||||
}
|
||||
|
||||
// Next returns the next value in the bitmap.
|
||||
|
|
@ -914,14 +915,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
|
|||
}
|
||||
|
||||
r := itr.c.runs[itr.j]
|
||||
runLength := int(r.last - r.start)
|
||||
runLength := int32(r.last - r.start)
|
||||
|
||||
if itr.k >= runLength {
|
||||
// Reached end of run, move to the next run.
|
||||
itr.j, itr.k = itr.j+1, -1
|
||||
}
|
||||
|
||||
if itr.j >= len(itr.c.runs) {
|
||||
if itr.j >= int32(len(itr.c.runs)) {
|
||||
// Reached end of runs, move to the next container.
|
||||
if !itr.citer.Next() {
|
||||
itr.c = nil
|
||||
|
|
@ -942,7 +943,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
|
|||
// Find first non-zero bit in current bitmap, if possible.
|
||||
hb := itr.j >> 6
|
||||
|
||||
if hb >= len(itr.c.bitmap) {
|
||||
if hb >= int32(len(itr.c.bitmap)) {
|
||||
if !itr.citer.Next() {
|
||||
itr.c = nil
|
||||
return 0, true
|
||||
|
|
@ -953,14 +954,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
|
|||
}
|
||||
lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64)
|
||||
if lb != 0 {
|
||||
itr.j = itr.j + trailingZeroN(lb)
|
||||
itr.j = itr.j + int32(trailingZeroN(lb))
|
||||
return itr.peek(), false
|
||||
}
|
||||
|
||||
// Otherwise iterate through remaining bitmaps to find next bit.
|
||||
for hb++; hb < len(itr.c.bitmap); hb++ {
|
||||
for hb++; hb < int32(len(itr.c.bitmap)); hb++ {
|
||||
if itr.c.bitmap[hb] != 0 {
|
||||
itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb])
|
||||
itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap[hb]))
|
||||
return itr.peek(), false
|
||||
}
|
||||
}
|
||||
|
|
@ -1008,7 +1009,7 @@ const runMaxSize = 2048
|
|||
type Container struct {
|
||||
mapped bool // mapped directly to a byte slice when true
|
||||
containerType byte // array, bitmap, or run
|
||||
n int // number of integers in container
|
||||
n int32 // number of integers in container
|
||||
array []uint16 // used for array containers
|
||||
bitmap []uint64 // used for bitmap containers
|
||||
runs []interval16 // used for RLE containers
|
||||
|
|
@ -1020,8 +1021,8 @@ type interval16 struct {
|
|||
}
|
||||
|
||||
// runlen returns the count of integers in the interval.
|
||||
func (iv interval16) runlen() int {
|
||||
return 1 + int(iv.last-iv.start)
|
||||
func (iv interval16) runlen() int32 {
|
||||
return 1 + int32(iv.last-iv.start)
|
||||
}
|
||||
|
||||
// newContainer returns a new instance of container.
|
||||
|
|
@ -1035,12 +1036,12 @@ func (c *Container) Mapped() bool {
|
|||
}
|
||||
|
||||
// N returns the cached bit count of the container
|
||||
func (c *Container) N() int {
|
||||
func (c *Container) N() int32 {
|
||||
return c.n
|
||||
}
|
||||
|
||||
// Update updates the container
|
||||
func (c *Container) Update(containerType byte, n int, mapped bool) {
|
||||
func (c *Container) Update(containerType byte, n int32, mapped bool) {
|
||||
c.containerType = containerType
|
||||
c.n = n
|
||||
c.mapped = mapped
|
||||
|
|
@ -1088,12 +1089,12 @@ func (c *Container) unmap() {
|
|||
}
|
||||
|
||||
// count counts all bits in the container.
|
||||
func (c *Container) count() (n int) {
|
||||
func (c *Container) count() (n int32) {
|
||||
return c.countRange(0, maxContainerVal+1)
|
||||
}
|
||||
|
||||
// countRange counts the number of bits set between [start, end).
|
||||
func (c *Container) countRange(start, end int) (n int) {
|
||||
func (c *Container) countRange(start, end int32) (n int32) {
|
||||
if c.isArray() {
|
||||
return c.arrayCountRange(start, end)
|
||||
} else if c.isRun() {
|
||||
|
|
@ -1102,10 +1103,10 @@ func (c *Container) countRange(start, end int) (n int) {
|
|||
return c.bitmapCountRange(start, end)
|
||||
}
|
||||
|
||||
func (c *Container) arrayCountRange(start, end int) (n int) {
|
||||
i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start })
|
||||
for ; i < len(c.array); i++ {
|
||||
v := int(c.array[i])
|
||||
func (c *Container) arrayCountRange(start, end int32) (n int32) {
|
||||
i := int32(sort.Search(len(c.array), func(i int) bool { return int32(c.array[i]) >= start }))
|
||||
for ; i < int32(len(c.array)); i++ {
|
||||
v := int32(c.array[i])
|
||||
if v >= end {
|
||||
break
|
||||
}
|
||||
|
|
@ -1114,14 +1115,14 @@ func (c *Container) arrayCountRange(start, end int) (n int) {
|
|||
return n
|
||||
}
|
||||
|
||||
func (c *Container) bitmapCountRange(start, end int) int {
|
||||
func (c *Container) bitmapCountRange(start, end int32) int32 {
|
||||
var n uint64
|
||||
i, j := start/64, end/64
|
||||
// Special case when start and end fall in the same word.
|
||||
if i == j {
|
||||
offi, offj := uint(start%64), uint(64-end%64)
|
||||
n += popcount((c.bitmap[i] >> offi) << (offj + offi))
|
||||
return int(n)
|
||||
return int32(n)
|
||||
}
|
||||
|
||||
// Count partial starting word.
|
||||
|
|
@ -1136,39 +1137,39 @@ func (c *Container) bitmapCountRange(start, end int) int {
|
|||
}
|
||||
|
||||
// Count partial ending word.
|
||||
if j < len(c.bitmap) {
|
||||
if j < int32(len(c.bitmap)) {
|
||||
off := 64 - (uint(end) % 64)
|
||||
n += popcount(c.bitmap[j] << off)
|
||||
}
|
||||
|
||||
return int(n)
|
||||
return int32(n)
|
||||
}
|
||||
|
||||
func (c *Container) runCountRange(start, end int) (n int) {
|
||||
func (c *Container) runCountRange(start, end int32) (n int32) {
|
||||
for _, iv := range c.runs {
|
||||
// iv is before range
|
||||
if int(iv.last) < start {
|
||||
if int32(iv.last) < start {
|
||||
continue
|
||||
}
|
||||
// iv is after range
|
||||
if end < int(iv.start) {
|
||||
if end < int32(iv.start) {
|
||||
break
|
||||
}
|
||||
// iv is superset of range
|
||||
if int(iv.start) < start && int(iv.last) > end {
|
||||
if int32(iv.start) < start && int32(iv.last) > end {
|
||||
return end - start
|
||||
}
|
||||
// iv is subset of range
|
||||
if int(iv.start) >= start && int(iv.last) < end {
|
||||
if int32(iv.start) >= start && int32(iv.last) < end {
|
||||
n += iv.runlen()
|
||||
}
|
||||
// iv overlaps beginning of range
|
||||
if int(iv.start) < start && int(iv.last) < end {
|
||||
n += int(iv.last) - start + 1
|
||||
if int32(iv.start) < start && int32(iv.last) < end {
|
||||
n += int32(iv.last) - start + 1
|
||||
}
|
||||
// iv overlaps end of range
|
||||
if int(iv.start) > start && int(iv.last) >= end {
|
||||
n += end - int(iv.start)
|
||||
if int32(iv.start) > start && int32(iv.last) >= end {
|
||||
n += end - int32(iv.start)
|
||||
}
|
||||
}
|
||||
return n
|
||||
|
|
@ -1286,34 +1287,34 @@ func (c *Container) Contains(v uint16) bool {
|
|||
}
|
||||
}
|
||||
|
||||
func (c *Container) bitmapCountRuns() (r int) {
|
||||
func (c *Container) bitmapCountRuns() (r int32) {
|
||||
for i := 0; i < 1023; i++ {
|
||||
v, v1 := c.bitmap[i], c.bitmap[i+1]
|
||||
r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1))
|
||||
r = r + int32(popcount((v<<1)&^v)+((v>>63)&^v1))
|
||||
}
|
||||
vl := c.bitmap[len(c.bitmap)-1]
|
||||
r = r + int(popcount((vl<<1)&^vl)+vl>>63)
|
||||
r = r + int32(popcount((vl<<1)&^vl)+vl>>63)
|
||||
return r
|
||||
}
|
||||
|
||||
func (c *Container) arrayCountRuns() (r int) {
|
||||
prev := -2
|
||||
func (c *Container) arrayCountRuns() (r int32) {
|
||||
prev := int32(-2)
|
||||
for _, v := range c.array {
|
||||
if prev+1 != int(v) {
|
||||
if prev+1 != int32(v) {
|
||||
r++
|
||||
}
|
||||
prev = int(v)
|
||||
prev = int32(v)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (c *Container) countRuns() (r int) {
|
||||
func (c *Container) countRuns() (r int32) {
|
||||
if c.isArray() {
|
||||
return c.arrayCountRuns()
|
||||
} else if c.isBitmap() {
|
||||
return c.bitmapCountRuns()
|
||||
} else if c.isRun() {
|
||||
return len(c.runs)
|
||||
return int32(len(c.runs))
|
||||
}
|
||||
|
||||
// sure hope this never happens
|
||||
|
|
@ -1369,10 +1370,10 @@ func (c *Container) bitmapContains(v uint16) bool {
|
|||
|
||||
// binSearchRuns returns the index of the run containing v, and true, when v is contained;
|
||||
// or the index of the next run starting after v, and false, when v is not contained.
|
||||
func binSearchRuns(v uint16, a []interval16) (int, bool) {
|
||||
i := sort.Search(len(a),
|
||||
func(i int) bool { return a[i].last >= v })
|
||||
if i < len(a) {
|
||||
func binSearchRuns(v uint16, a []interval16) (int32, bool) {
|
||||
i := int32(sort.Search(len(a),
|
||||
func(i int) bool { return a[i].last >= v }))
|
||||
if i < int32(len(a)) {
|
||||
return i, (v >= a[i].start) && (v <= a[i].last)
|
||||
}
|
||||
|
||||
|
|
@ -1698,7 +1699,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
|
|||
// assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
|
||||
//}
|
||||
|
||||
// Write sizeof(uint32) * cardinality bytes.
|
||||
// Write sizeof(uint16) * cardinality bytes.
|
||||
nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n])
|
||||
return int64(nn), err
|
||||
}
|
||||
|
|
@ -1767,7 +1768,7 @@ func (c *Container) check() error {
|
|||
var a ErrorList
|
||||
|
||||
if c.isArray() {
|
||||
if len(c.array) != c.n {
|
||||
if int32(len(c.array)) != c.n {
|
||||
a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n))
|
||||
}
|
||||
} else if c.isRun() {
|
||||
|
|
@ -1796,7 +1797,7 @@ func (c *Container) check() error {
|
|||
type containerInfo struct {
|
||||
Key uint64 // container key
|
||||
Type string // container type (array, bitmap, or run)
|
||||
N int // number of bits
|
||||
N int32 // number of bits
|
||||
Alloc int // memory used
|
||||
Pointer unsafe.Pointer // offset within the mmap
|
||||
}
|
||||
|
|
@ -1838,7 +1839,7 @@ func flipRun(b *Container) *Container {
|
|||
return flipBitmap(x)
|
||||
}
|
||||
|
||||
func intersectionCount(a, b *Container) int {
|
||||
func intersectionCount(a, b *Container) int32 {
|
||||
if a.isArray() {
|
||||
if b.isArray() {
|
||||
return intersectionCountArrayArray(a, b)
|
||||
|
|
@ -1866,7 +1867,7 @@ func intersectionCount(a, b *Container) int {
|
|||
}
|
||||
}
|
||||
|
||||
func intersectionCountArrayArray(a, b *Container) (n int) {
|
||||
func intersectionCountArrayArray(a, b *Container) (n int32) {
|
||||
na, nb := len(a.array), len(b.array)
|
||||
for i, j := 0, 0; i < na && j < nb; {
|
||||
va, vb := a.array[i], b.array[j]
|
||||
|
|
@ -1882,7 +1883,7 @@ func intersectionCountArrayArray(a, b *Container) (n int) {
|
|||
return n
|
||||
}
|
||||
|
||||
func intersectionCountArrayRun(a, b *Container) (n int) {
|
||||
func intersectionCountArrayRun(a, b *Container) (n int32) {
|
||||
na, nb := len(a.array), len(b.runs)
|
||||
for i, j := 0, 0; i < na && j < nb; {
|
||||
va, vb := a.array[i], b.runs[j]
|
||||
|
|
@ -1898,7 +1899,7 @@ func intersectionCountArrayRun(a, b *Container) (n int) {
|
|||
return n
|
||||
}
|
||||
|
||||
func intersectionCountRunRun(a, b *Container) (n int) {
|
||||
func intersectionCountRunRun(a, b *Container) (n int32) {
|
||||
na, nb := len(a.runs), len(b.runs)
|
||||
for i, j := 0, 0; i < na && j < nb; {
|
||||
va, vb := a.runs[i], b.runs[j]
|
||||
|
|
@ -1910,33 +1911,33 @@ func intersectionCountRunRun(a, b *Container) (n int) {
|
|||
j++
|
||||
} else if va.last > vb.last && va.start >= vb.start {
|
||||
// |--vb-|-|-va--|
|
||||
n += 1 + int(vb.last-va.start)
|
||||
n += 1 + int32(vb.last-va.start)
|
||||
j++
|
||||
} else if va.last > vb.last && va.start < vb.start {
|
||||
// |--va|--vb--|--|
|
||||
n += 1 + int(vb.last-vb.start)
|
||||
n += 1 + int32(vb.last-vb.start)
|
||||
j++
|
||||
} else if va.last <= vb.last && va.start >= vb.start {
|
||||
// |--vb|--va--|--|
|
||||
n += 1 + int(va.last-va.start)
|
||||
n += 1 + int32(va.last-va.start)
|
||||
i++
|
||||
} else if va.last <= vb.last && va.start < vb.start {
|
||||
// |--va-|-|-vb--|
|
||||
n += 1 + int(va.last-vb.start)
|
||||
n += 1 + int32(va.last-vb.start)
|
||||
i++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func intersectionCountBitmapRun(a, b *Container) (n int) {
|
||||
func intersectionCountBitmapRun(a, b *Container) (n int32) {
|
||||
for _, iv := range b.runs {
|
||||
n += a.bitmapCountRange(int(iv.start), int(iv.last)+1)
|
||||
n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func intersectionCountArrayBitmap(a, b *Container) (n int) {
|
||||
func intersectionCountArrayBitmap(a, b *Container) (n int32) {
|
||||
ln := len(b.bitmap)
|
||||
for _, val := range a.array {
|
||||
i := int(val >> 6)
|
||||
|
|
@ -1944,13 +1945,13 @@ func intersectionCountArrayBitmap(a, b *Container) (n int) {
|
|||
break
|
||||
}
|
||||
off := val % 64
|
||||
n += int(b.bitmap[i]>>off) & 1
|
||||
n += int32(b.bitmap[i]>>off) & 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func intersectionCountBitmapBitmap(a, b *Container) (n int) {
|
||||
return int(popcountAndSlice(a.bitmap, b.bitmap))
|
||||
func intersectionCountBitmapBitmap(a, b *Container) (n int32) {
|
||||
return int32(popcountAndSlice(a.bitmap, b.bitmap))
|
||||
}
|
||||
|
||||
func intersect(a, b *Container) *Container {
|
||||
|
|
@ -1995,7 +1996,7 @@ func intersectArrayArray(a, b *Container) *Container {
|
|||
i, j = i+1, j+1
|
||||
}
|
||||
}
|
||||
output.n = len(output.array)
|
||||
output.n = int32(len(output.array))
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2016,7 +2017,7 @@ func intersectArrayRun(a, b *Container) *Container {
|
|||
i++
|
||||
}
|
||||
}
|
||||
output.n = len(output.array)
|
||||
output.n = int32(len(output.array))
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2050,7 +2051,7 @@ func intersectRunRun(a, b *Container) *Container {
|
|||
i++
|
||||
}
|
||||
}
|
||||
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
|
||||
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > runMaxSize {
|
||||
output.runToBitmap()
|
||||
|
|
@ -2076,7 +2077,7 @@ func intersectBitmapRun(a, b *Container) *Container {
|
|||
}
|
||||
}
|
||||
}
|
||||
output.n = len(output.array)
|
||||
output.n = int32(len(output.array))
|
||||
} else {
|
||||
// right now this iterates through the runs and sets integers in the
|
||||
// bitmap that are in the runs. alternately, we could zero out ranges in
|
||||
|
|
@ -2093,22 +2094,22 @@ func intersectBitmapRun(a, b *Container) *Container {
|
|||
for valast >= vb.start && vastart <= vb.last && i < bitmapN {
|
||||
if vastart >= vb.start && valast <= vb.last { // a within b
|
||||
output.bitmap[i] = a.bitmap[i]
|
||||
output.n += int(popcount(a.bitmap[i]))
|
||||
output.n += int32(popcount(a.bitmap[i]))
|
||||
} else if vb.start >= vastart && vb.last <= valast { // b within a
|
||||
var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
|
||||
bits := a.bitmap[i] & mask
|
||||
output.bitmap[i] |= bits
|
||||
output.n += int(popcount(bits))
|
||||
output.n += int32(popcount(bits))
|
||||
} else if vastart < vb.start { // a overlaps front of b
|
||||
offset := 64 - (1 + valast - vb.start)
|
||||
bits := (a.bitmap[i] >> offset) << offset
|
||||
output.bitmap[i] |= bits
|
||||
output.n += int(popcount(bits))
|
||||
output.n += int32(popcount(bits))
|
||||
} else if vb.start < vastart { // b overlaps front of a
|
||||
offset := 64 - (1 + vb.last - vastart)
|
||||
bits := (a.bitmap[i] << offset) >> offset
|
||||
output.bitmap[i] |= bits
|
||||
output.n += int(popcount(bits))
|
||||
output.n += int32(popcount(bits))
|
||||
}
|
||||
// update loop vars
|
||||
i++
|
||||
|
|
@ -2134,20 +2135,30 @@ func intersectArrayBitmap(a, b *Container) *Container {
|
|||
output.array = append(output.array, va)
|
||||
}
|
||||
}
|
||||
output.n = len(output.array)
|
||||
output.n = int32(len(output.array))
|
||||
return output
|
||||
}
|
||||
|
||||
func intersectBitmapBitmap(a, b *Container) *Container {
|
||||
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
|
||||
|
||||
for i := range a.bitmap {
|
||||
v := a.bitmap[i] & b.bitmap[i]
|
||||
output.bitmap[i] = v
|
||||
output.n += int(popcount(v))
|
||||
|
||||
// local variables added to prevent BCE checks in loop
|
||||
// see https://go101.org/article/bounds-check-elimination.html
|
||||
var (
|
||||
ab = a.bitmap[:bitmapN]
|
||||
bb = b.bitmap[:bitmapN]
|
||||
buf = make([]uint64, bitmapN)
|
||||
ob = buf[:bitmapN]
|
||||
n int32
|
||||
)
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
ob[i] = ab[i] & bb[i]
|
||||
n += int32(popcount(ob[i]))
|
||||
}
|
||||
|
||||
output := &Container{
|
||||
bitmap: ob,
|
||||
n: n,
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
output.optimize()
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2249,10 +2260,10 @@ func unionArrayRun(a, b *Container) *Container {
|
|||
// interval is earlier than the start of the last interval in the list of runs.
|
||||
// Its return value is the amount by which the cardinality of the container was
|
||||
// increased.
|
||||
func (c *Container) runAppendInterval(v interval16) int {
|
||||
func (c *Container) runAppendInterval(v interval16) int32 {
|
||||
if len(c.runs) == 0 {
|
||||
c.runs = append(c.runs, v)
|
||||
return int(v.last-v.start) + 1
|
||||
return int32(v.last-v.start) + 1
|
||||
}
|
||||
|
||||
last := c.runs[len(c.runs)-1]
|
||||
|
|
@ -2261,10 +2272,10 @@ func (c *Container) runAppendInterval(v interval16) int {
|
|||
}
|
||||
if last.last+1 >= v.start && v.last > last.last {
|
||||
c.runs[len(c.runs)-1].last = v.last
|
||||
return int(v.last - last.last)
|
||||
return int32(v.last - last.last)
|
||||
} else if last.last+1 < v.start {
|
||||
c.runs = append(c.runs, v)
|
||||
return int(v.last-v.start) + 1
|
||||
return int32(v.last-v.start) + 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -2328,16 +2339,16 @@ func (c *Container) bitmapSetRange(i, j uint64) {
|
|||
xcnt := popcount(X)
|
||||
ycnt := popcount(Y)
|
||||
if x == y {
|
||||
c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y)))
|
||||
c.n += int32((j - i) - popcount(c.bitmap[x]&(X&Y)))
|
||||
c.bitmap[x] |= (X & Y)
|
||||
} else {
|
||||
c.n += int(xcnt - popcount(c.bitmap[x]&X))
|
||||
c.n += int32(xcnt - popcount(c.bitmap[x]&X))
|
||||
c.bitmap[x] |= X
|
||||
for i := x + 1; i < y; i++ {
|
||||
c.n += int(64 - popcount(c.bitmap[i]))
|
||||
c.n += int32(64 - popcount(c.bitmap[i]))
|
||||
c.bitmap[i] = maxBitmap
|
||||
}
|
||||
c.n += int(ycnt - popcount(c.bitmap[y]&Y))
|
||||
c.n += int32(ycnt - popcount(c.bitmap[y]&Y))
|
||||
c.bitmap[y] |= Y
|
||||
}
|
||||
}
|
||||
|
|
@ -2351,19 +2362,19 @@ func (c *Container) bitmapXorRange(i, j uint64) {
|
|||
if x == y {
|
||||
cnt := popcount(c.bitmap[x])
|
||||
c.bitmap[x] ^= (X & Y) //// flip
|
||||
c.n += int(popcount(c.bitmap[x]) - cnt)
|
||||
c.n += int32(popcount(c.bitmap[x]) - cnt)
|
||||
} else {
|
||||
cnt := popcount(c.bitmap[x])
|
||||
c.bitmap[x] ^= X
|
||||
c.n += int(popcount(c.bitmap[x]) - cnt)
|
||||
c.n += int32(popcount(c.bitmap[x]) - cnt)
|
||||
for i := x + 1; i < y; i++ {
|
||||
cnt = popcount(c.bitmap[i])
|
||||
c.bitmap[i] ^= maxBitmap
|
||||
c.n += int(popcount(c.bitmap[i]) - cnt)
|
||||
c.n += int32(popcount(c.bitmap[i]) - cnt)
|
||||
}
|
||||
cnt = popcount(c.bitmap[y])
|
||||
c.bitmap[y] ^= Y
|
||||
c.n += int(popcount(c.bitmap[y]) - cnt)
|
||||
c.n += int32(popcount(c.bitmap[y]) - cnt)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2374,16 +2385,16 @@ func (c *Container) bitmapZeroRange(i, j uint64) {
|
|||
var X uint64 = maxBitmap << (i % 64)
|
||||
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
|
||||
if x == y {
|
||||
c.n -= int(popcount(c.bitmap[x] & (X & Y)))
|
||||
c.n -= int32(popcount(c.bitmap[x] & (X & Y)))
|
||||
c.bitmap[x] &= ^(X & Y)
|
||||
} else {
|
||||
c.n -= int(popcount(c.bitmap[x] & X))
|
||||
c.n -= int32(popcount(c.bitmap[x] & X))
|
||||
c.bitmap[x] &= ^X
|
||||
for i := x + 1; i < y; i++ {
|
||||
c.n -= int(popcount(c.bitmap[i]))
|
||||
c.n -= int32(popcount(c.bitmap[i]))
|
||||
c.bitmap[i] = 0
|
||||
}
|
||||
c.n -= int(popcount(c.bitmap[y] & Y))
|
||||
c.n -= int32(popcount(c.bitmap[y] & Y))
|
||||
c.bitmap[y] &= ^Y
|
||||
}
|
||||
}
|
||||
|
|
@ -2437,17 +2448,28 @@ func unionArrayBitmap(a, b *Container) *Container {
|
|||
}
|
||||
|
||||
func unionBitmapBitmap(a, b *Container) *Container {
|
||||
output := &Container{
|
||||
bitmap: make([]uint64, bitmapN),
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
// local variables added to prevent BCE checks in loop
|
||||
// see https://go101.org/article/bounds-check-elimination.html
|
||||
|
||||
var (
|
||||
ab = a.bitmap[:bitmapN]
|
||||
bb = b.bitmap[:bitmapN]
|
||||
buf = make([]uint64, bitmapN)
|
||||
ob = buf[:bitmapN]
|
||||
|
||||
n int32
|
||||
)
|
||||
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
v := a.bitmap[i] | b.bitmap[i]
|
||||
output.bitmap[i] = v
|
||||
output.n += int(popcount(v))
|
||||
ob[i] = ab[i] | bb[i]
|
||||
n += int32(popcount(ob[i]))
|
||||
}
|
||||
|
||||
output := &Container{
|
||||
bitmap: ob,
|
||||
n: n,
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2519,7 +2541,7 @@ func differenceArrayRun(a, b *Container) *Container {
|
|||
j := 0 // run index
|
||||
|
||||
// handle overlap
|
||||
for i < a.n {
|
||||
for i < int(a.n) {
|
||||
|
||||
// keep all array elements before beginning of runs
|
||||
if a.array[i] < b.runs[j].start {
|
||||
|
|
@ -2551,7 +2573,7 @@ func differenceArrayRun(a, b *Container) *Container {
|
|||
output.array = append(output.array, a.array[i:]...)
|
||||
// TODO: consider handling container.n mutations in one place
|
||||
// like we do with container.add().
|
||||
output.n += len(a.array[i:])
|
||||
output.n += int32(len(a.array[i:]))
|
||||
} else {
|
||||
for _, v := range a.array[i:] {
|
||||
output.add(v)
|
||||
|
|
@ -2609,7 +2631,7 @@ RUNLOOP:
|
|||
continue
|
||||
}
|
||||
output.runs = append(output.runs, interval16{start: start, last: vb - 1})
|
||||
output.n += int(vb - start)
|
||||
output.n += int32(vb - start)
|
||||
if vb == 65535 { // overflow
|
||||
break RUNLOOP
|
||||
}
|
||||
|
|
@ -2623,7 +2645,7 @@ RUNLOOP:
|
|||
|
||||
if start <= run.last {
|
||||
output.runs = append(output.runs, interval16{start: start, last: run.last})
|
||||
output.n += int(run.last - start + 1)
|
||||
output.n += int32(run.last - start + 1)
|
||||
}
|
||||
}
|
||||
output.optimize()
|
||||
|
|
@ -2679,7 +2701,7 @@ func differenceRunBitmap(a, b *Container) *Container {
|
|||
}
|
||||
}
|
||||
|
||||
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
|
||||
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > runMaxSize {
|
||||
output.runToBitmap()
|
||||
|
|
@ -2763,7 +2785,7 @@ func differenceArrayBitmap(a, b *Container) *Container {
|
|||
output.array = append(output.array, va)
|
||||
}
|
||||
}
|
||||
output.n = len(output.array)
|
||||
output.n = int32(len(output.array))
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -2783,13 +2805,27 @@ func differenceBitmapArray(a, b *Container) *Container {
|
|||
}
|
||||
|
||||
func differenceBitmapBitmap(a, b *Container) *Container {
|
||||
output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
|
||||
// local variables added to prevent BCE checks in loop
|
||||
// see https://go101.org/article/bounds-check-elimination.html
|
||||
|
||||
for i := range a.bitmap {
|
||||
v := a.bitmap[i] & (^b.bitmap[i])
|
||||
output.bitmap[i] = v
|
||||
output.n += int(popcount(v))
|
||||
var (
|
||||
ab = a.bitmap[:bitmapN]
|
||||
bb = b.bitmap[:bitmapN]
|
||||
buf = make([]uint64, bitmapN)
|
||||
ob = buf[:bitmapN]
|
||||
|
||||
n int32
|
||||
)
|
||||
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
ob[i] = ab[i] & (^bb[i])
|
||||
n += int32(popcount(ob[i]))
|
||||
}
|
||||
|
||||
output := &Container{
|
||||
bitmap: ob,
|
||||
n: n,
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
if output.n < ArrayMaxSize {
|
||||
output.bitmapToArray()
|
||||
|
|
@ -2874,16 +2910,28 @@ func xorArrayBitmap(a, b *Container) *Container {
|
|||
}
|
||||
|
||||
func xorBitmapBitmap(a, b *Container) *Container {
|
||||
output := &Container{
|
||||
bitmap: make([]uint64, bitmapN),
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
// local variables added to prevent BCE checks in loop
|
||||
// see https://go101.org/article/bounds-check-elimination.html
|
||||
|
||||
var (
|
||||
ab = a.bitmap[:bitmapN]
|
||||
bb = b.bitmap[:bitmapN]
|
||||
buf = make([]uint64, bitmapN)
|
||||
ob = buf[:bitmapN]
|
||||
|
||||
n int32
|
||||
)
|
||||
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
v := a.bitmap[i] ^ b.bitmap[i]
|
||||
output.bitmap[i] = v
|
||||
output.n += int(popcount(v))
|
||||
ob[i] = ab[i] ^ bb[i]
|
||||
n += int32(popcount(ob[i]))
|
||||
}
|
||||
|
||||
output := &Container{
|
||||
bitmap: ob,
|
||||
n: n,
|
||||
containerType: containerBitmap,
|
||||
}
|
||||
if output.count() < ArrayMaxSize {
|
||||
output.bitmapToArray()
|
||||
}
|
||||
|
|
@ -2962,9 +3010,9 @@ func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
|
|||
|
||||
// search32 returns the index of value in a. If value is not found, it works the
|
||||
// same way as search64.
|
||||
func search32(a []uint16, value uint16) int {
|
||||
func search32(a []uint16, value uint16) int32 {
|
||||
// Optimize for elements and the last element.
|
||||
n := len(a)
|
||||
n := int32(len(a))
|
||||
if n == 0 {
|
||||
return -1
|
||||
} else if a[n-1] == value {
|
||||
|
|
@ -2972,9 +3020,9 @@ func search32(a []uint16, value uint16) int {
|
|||
}
|
||||
|
||||
// Otherwise perform binary search for exact match.
|
||||
lo, hi := 0, n-1
|
||||
lo, hi := int32(0), n-1
|
||||
for lo+16 <= hi {
|
||||
i := int(uint((lo + hi)) >> 1)
|
||||
i := int32(uint((lo + hi)) >> 1)
|
||||
v := a[i]
|
||||
|
||||
if v < value {
|
||||
|
|
@ -3280,7 +3328,7 @@ func xorRunRun(a, b *Container) *Container {
|
|||
|
||||
}
|
||||
|
||||
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
|
||||
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > runMaxSize {
|
||||
output.runToBitmap()
|
||||
|
|
@ -3295,7 +3343,7 @@ func xorBitmapRun(a, b *Container) *Container {
|
|||
output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
|
||||
}
|
||||
|
||||
if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
|
||||
if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 {
|
||||
output.runToArray()
|
||||
} else if len(output.runs) > runMaxSize {
|
||||
output.runToBitmap()
|
||||
|
|
@ -3336,9 +3384,14 @@ func popcount(x uint64) uint64 {
|
|||
}
|
||||
|
||||
func popcountAndSlice(s, m []uint64) uint64 {
|
||||
var (
|
||||
a = s[:bitmapN]
|
||||
b = m[:bitmapN]
|
||||
)
|
||||
|
||||
cnt := uint64(0)
|
||||
for i := range s {
|
||||
cnt += popcount(s[i] & m[i])
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
cnt += popcount(a[i] & b[i])
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
|
@ -3494,7 +3547,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) {
|
|||
c.runs = nil
|
||||
c.bitmap = nil
|
||||
c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n]
|
||||
pos += c.n * 2
|
||||
pos += int(c.n * 2)
|
||||
case containerBitmap:
|
||||
c.array = nil
|
||||
c.runs = nil
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ func TestRunAppendInterval(t *testing.T) {
|
|||
tests := []struct {
|
||||
base []interval16
|
||||
app interval16
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{
|
||||
base: []interval16{},
|
||||
|
|
@ -207,10 +207,10 @@ func TestRunContains(t *testing.T) {
|
|||
func TestBitmapCountRange(t *testing.T) {
|
||||
c := Container{containerType: containerBitmap}
|
||||
tests := []struct {
|
||||
start int
|
||||
end int
|
||||
start int32
|
||||
end int32
|
||||
bitmap []uint64
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{start: 0, end: 1, bitmap: []uint64{1}, exp: 1},
|
||||
{start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2},
|
||||
|
|
@ -251,7 +251,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) {
|
|||
b.bitmapToRun()
|
||||
res = intersectRunRun(a, b)
|
||||
n := intersectionCountRunRun(a, b)
|
||||
if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) {
|
||||
if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) {
|
||||
t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1)
|
||||
}
|
||||
}
|
||||
|
|
@ -261,7 +261,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
|
|||
tests := []struct {
|
||||
array []uint16
|
||||
bitmap []uint64
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{
|
||||
array: []uint16{0},
|
||||
|
|
@ -384,7 +384,7 @@ func TestIntersectionCountRunRun(t *testing.T) {
|
|||
tests := []struct {
|
||||
aruns []interval16
|
||||
bruns []interval16
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{
|
||||
aruns: []interval16{},
|
||||
|
|
@ -478,7 +478,7 @@ func TestIntersectRunRun(t *testing.T) {
|
|||
aruns []interval16
|
||||
bruns []interval16
|
||||
exp []interval16
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
aruns: []interval16{},
|
||||
|
|
@ -540,7 +540,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
|
|||
bitmap []uint64
|
||||
runs []interval16
|
||||
exp []uint64
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{1},
|
||||
|
|
@ -602,7 +602,7 @@ func TestIntersectBitmapRunArray(t *testing.T) {
|
|||
bitmap []uint64
|
||||
runs []interval16
|
||||
exp []uint16
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{1},
|
||||
|
|
@ -761,11 +761,11 @@ func TestDifferenceMixed(t *testing.T) {
|
|||
a.containerType = containerRun
|
||||
|
||||
b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
|
||||
b.n = len(b.array)
|
||||
b.n = int32(len(b.array))
|
||||
b.containerType = containerArray
|
||||
|
||||
d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
|
||||
d.n = len(d.array)
|
||||
d.n = int32(len(d.array))
|
||||
d.containerType = containerArray
|
||||
|
||||
res := difference(a, b)
|
||||
|
|
@ -784,11 +784,11 @@ func TestDifferenceMixed(t *testing.T) {
|
|||
t.Fatalf("test #3 expected empty but got %v", res.runs)
|
||||
}
|
||||
|
||||
c.bitmap = []uint64{0x64}
|
||||
c.bitmap = MakeBitmap([]uint64{0x64})
|
||||
c.n = c.countRange(0, 100)
|
||||
c.containerType = containerBitmap
|
||||
res = difference(c, a)
|
||||
if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
|
||||
if !reflect.DeepEqual(res.bitmap, MakeBitmap([]uint64{0x4})) {
|
||||
t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
|
||||
}
|
||||
|
||||
|
|
@ -939,7 +939,7 @@ func TestBitmapSetRange(t *testing.T) {
|
|||
start uint64
|
||||
last uint64
|
||||
exp []uint64
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{0x0000000000FFF900},
|
||||
|
|
@ -991,7 +991,7 @@ func TestArrayToBitmap(t *testing.T) {
|
|||
copy(exp, test.exp)
|
||||
|
||||
a.array = test.array
|
||||
a.n = len(test.array)
|
||||
a.n = int32(len(test.array))
|
||||
a.arrayToBitmap()
|
||||
if !reflect.DeepEqual(a.bitmap, exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
|
||||
|
|
@ -1016,10 +1016,10 @@ func TestBitmapToArray(t *testing.T) {
|
|||
}
|
||||
for i, test := range tests {
|
||||
a.bitmap = make([]uint64, bitmapN)
|
||||
n := 0
|
||||
n := int32(0)
|
||||
for i, v := range test.bitmap {
|
||||
a.bitmap[i] = v
|
||||
n += int(popcount(v))
|
||||
n += int32(popcount(v))
|
||||
}
|
||||
a.n = n
|
||||
|
||||
|
|
@ -1067,7 +1067,7 @@ func TestRunToBitmap(t *testing.T) {
|
|||
}
|
||||
|
||||
a.runs = test.runs
|
||||
a.n = n
|
||||
a.n = int32(n)
|
||||
a.runToBitmap()
|
||||
if !reflect.DeepEqual(a.bitmap, exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
|
||||
|
|
@ -1149,7 +1149,7 @@ func TestBitmapToRun(t *testing.T) {
|
|||
a.bitmap[i] = v
|
||||
n += int(popcount(v))
|
||||
}
|
||||
a.n = n
|
||||
a.n = int32(n)
|
||||
x := a.bitmap
|
||||
a.bitmapToRun()
|
||||
if !reflect.DeepEqual(a.runs, test.exp) {
|
||||
|
|
@ -1188,7 +1188,7 @@ func TestArrayToRun(t *testing.T) {
|
|||
|
||||
for i, test := range tests {
|
||||
a.array = test.array
|
||||
a.n = int(len(test.array))
|
||||
a.n = int32(len(test.array))
|
||||
a.arrayToRun()
|
||||
if !reflect.DeepEqual(a.runs, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
|
||||
|
|
@ -1222,7 +1222,7 @@ func TestRunToArray(t *testing.T) {
|
|||
|
||||
for i, test := range tests {
|
||||
a.runs = test.runs
|
||||
a.n = len(test.exp)
|
||||
a.n = int32(len(test.exp))
|
||||
a.runToArray()
|
||||
if !reflect.DeepEqual(a.array, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
|
||||
|
|
@ -1237,7 +1237,7 @@ func TestBitmapZeroRange(t *testing.T) {
|
|||
start uint64
|
||||
last uint64
|
||||
exp []uint64
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{0x0000000000FFFF00},
|
||||
|
|
@ -1279,7 +1279,7 @@ func TestUnionBitmapRun(t *testing.T) {
|
|||
bitmap []uint64
|
||||
runs []interval16
|
||||
exp []uint64
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{2},
|
||||
|
|
@ -1313,7 +1313,7 @@ func TestBitmapCountRuns(t *testing.T) {
|
|||
c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)}
|
||||
tests := []struct {
|
||||
bitmap []uint64
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{0xFF00FF00},
|
||||
|
|
@ -1361,7 +1361,7 @@ func TestArrayCountRuns(t *testing.T) {
|
|||
c := &Container{containerType: containerArray}
|
||||
tests := []struct {
|
||||
array []uint16
|
||||
exp int
|
||||
exp int32
|
||||
}{
|
||||
{
|
||||
array: []uint16{},
|
||||
|
|
@ -1414,7 +1414,7 @@ func TestDifferenceArrayRun(t *testing.T) {
|
|||
}
|
||||
for i, test := range tests {
|
||||
a.array = test.array
|
||||
a.n = len(a.array)
|
||||
a.n = int32(len(a.array))
|
||||
b.runs = test.runs
|
||||
b.n = b.runCountRange(0, 100)
|
||||
ret := differenceArrayRun(a, b)
|
||||
|
|
@ -1482,7 +1482,7 @@ func TestDifferenceRunArray(t *testing.T) {
|
|||
a.runs = test.runs
|
||||
a.n = a.runCountRange(0, 100)
|
||||
b.array = test.array
|
||||
b.n = len(b.array)
|
||||
b.n = int32(len(b.array))
|
||||
ret := differenceRunArray(a, b)
|
||||
if !reflect.DeepEqual(ret.runs, test.exp) {
|
||||
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
|
||||
|
|
@ -1730,7 +1730,7 @@ func TestDifferenceRunRun(t *testing.T) {
|
|||
aruns []interval16
|
||||
bruns []interval16
|
||||
exp []interval16
|
||||
expn int
|
||||
expn int32
|
||||
}{
|
||||
{
|
||||
// this tests all six overlap combinations
|
||||
|
|
@ -2009,7 +2009,7 @@ func TestBitmapXorRange(t *testing.T) {
|
|||
start uint64
|
||||
last uint64
|
||||
exp []uint64
|
||||
expN int
|
||||
expN int32
|
||||
}{
|
||||
{
|
||||
bitmap: []uint64{0x0000000000000000},
|
||||
|
|
@ -2302,7 +2302,7 @@ func TestRunBinSearchContains(t *testing.T) {
|
|||
runs []interval16
|
||||
index uint16
|
||||
exp struct {
|
||||
index int
|
||||
index int32
|
||||
found bool
|
||||
}
|
||||
}{
|
||||
|
|
@ -2310,7 +2310,7 @@ func TestRunBinSearchContains(t *testing.T) {
|
|||
runs: []interval16{{start: 0, last: 10}},
|
||||
index: uint16(3),
|
||||
exp: struct {
|
||||
index int
|
||||
index int32
|
||||
found bool
|
||||
}{index: 0, found: true},
|
||||
},
|
||||
|
|
@ -2318,7 +2318,7 @@ func TestRunBinSearchContains(t *testing.T) {
|
|||
runs: []interval16{{start: 0, last: 10}},
|
||||
index: uint16(13),
|
||||
exp: struct {
|
||||
index int
|
||||
index int32
|
||||
found bool
|
||||
}{index: 0, found: false},
|
||||
},
|
||||
|
|
@ -2326,7 +2326,7 @@ func TestRunBinSearchContains(t *testing.T) {
|
|||
runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
|
||||
index: uint16(13),
|
||||
exp: struct {
|
||||
index int
|
||||
index int32
|
||||
found bool
|
||||
}{index: 0, found: false},
|
||||
},
|
||||
|
|
@ -2334,7 +2334,7 @@ func TestRunBinSearchContains(t *testing.T) {
|
|||
runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
|
||||
index: uint16(36),
|
||||
exp: struct {
|
||||
index int
|
||||
index int32
|
||||
found bool
|
||||
}{index: 1, found: false},
|
||||
},
|
||||
|
|
@ -2355,7 +2355,7 @@ func TestRunBinSearch(t *testing.T) {
|
|||
runs []interval16
|
||||
search uint16
|
||||
exp bool
|
||||
expi int
|
||||
expi int32
|
||||
}{
|
||||
{
|
||||
runs: []interval16{{2, 10}, {50, 60}, {80, 90}},
|
||||
|
|
|
|||
26
server.go
26
server.go
|
|
@ -234,6 +234,13 @@ func OptServerClusterHasher(h Hasher) ServerOption {
|
|||
}
|
||||
}
|
||||
|
||||
func OptServerTranslateFileMapSize(mapSize int) ServerOption {
|
||||
return func(s *Server) error {
|
||||
s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewServer returns a new instance of Server.
|
||||
func NewServer(opts ...ServerOption) (*Server, error) {
|
||||
s := &Server{
|
||||
|
|
@ -330,20 +337,20 @@ func (s *Server) Open() error {
|
|||
|
||||
// Initialize id-key storage.
|
||||
if err := s.holder.translateFile.Open(); err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "opening TranslateFile")
|
||||
}
|
||||
|
||||
// Open Cluster management.
|
||||
if err := s.cluster.waitForStarted(); err != nil {
|
||||
return fmt.Errorf("opening Cluster: %v", err)
|
||||
return errors.Wrap(err, "opening Cluster")
|
||||
}
|
||||
|
||||
// Open holder.
|
||||
if err := s.holder.Open(); err != nil {
|
||||
return fmt.Errorf("opening Holder: %v", err)
|
||||
return errors.Wrap(err, "opening Holder")
|
||||
}
|
||||
if err := s.cluster.setNodeState(nodeStateReady); err != nil {
|
||||
return fmt.Errorf("setting nodeState: %v", err)
|
||||
return errors.Wrap(err, "setting nodeState")
|
||||
}
|
||||
|
||||
// Listen for joining nodes.
|
||||
|
|
@ -474,7 +481,9 @@ func (s *Server) receiveMessage(m Message) error {
|
|||
if f == nil {
|
||||
return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field)
|
||||
}
|
||||
f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard))
|
||||
if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
|
||||
return errors.Wrap(err, "adding remote available shards")
|
||||
}
|
||||
case *CreateIndexMessage:
|
||||
opt := obj.Meta
|
||||
_, err := s.holder.CreateIndex(obj.Index, *opt)
|
||||
|
|
@ -633,13 +642,15 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
|
|||
for _, fs := range is.Fields {
|
||||
f := s.holder.Field(is.Name, fs.Name)
|
||||
|
||||
// if we don't know about an field locally, log a error because
|
||||
// if we don't know about a field locally, log an error because
|
||||
// fields should be created and synced prior to shard creation
|
||||
if f == nil {
|
||||
s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name)
|
||||
continue
|
||||
}
|
||||
f.addRemoteAvailableShards(fs.AvailableShards)
|
||||
if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil {
|
||||
return errors.Wrap(err, "adding remote available shards")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -664,6 +675,7 @@ func (s *Server) monitorDiagnostics() {
|
|||
s.diagnostics.Set("NumCPU", runtime.NumCPU())
|
||||
s.diagnostics.Set("NodeID", s.nodeID)
|
||||
s.diagnostics.Set("ClusterID", s.cluster.id)
|
||||
s.diagnostics.EnrichWithCPUInfo()
|
||||
s.diagnostics.EnrichWithOSInfo()
|
||||
|
||||
// Flush the diagnostics metrics at startup, then on each tick interval
|
||||
|
|
|
|||
|
|
@ -71,8 +71,9 @@ type Config struct {
|
|||
// Gossip config is based around memberlist.Config.
|
||||
Gossip gossip.Config `toml:"gossip"`
|
||||
|
||||
// DEPRECATED: Translation config supports translation store replication.
|
||||
Translation struct {
|
||||
MapSize int `toml:"map-size"`
|
||||
// DEPRECATED: Translation config supports translation store replication.
|
||||
PrimaryURL string `toml:"primary-url"`
|
||||
} `toml:"translation"`
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) {
|
|||
|
||||
})
|
||||
|
||||
t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) {
|
||||
// Roaring import into a non-set field should fail.
|
||||
if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
|
||||
req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData))
|
||||
req.Header.Set("Content-Type", "application/x-binary")
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != gohttp.StatusBadRequest {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
t.Run("Status", func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
|
||||
|
|
|
|||
|
|
@ -283,6 +283,14 @@ func (m *Command) SetupServer() error {
|
|||
coordinatorOpt,
|
||||
}
|
||||
|
||||
if m.Config.Translation.MapSize > 0 {
|
||||
serverOptions = append(
|
||||
serverOptions,
|
||||
pilosa.OptServerTranslateFileMapSize(
|
||||
m.Config.Translation.MapSize,
|
||||
),
|
||||
)
|
||||
}
|
||||
serverOptions = append(serverOptions, m.serverOptions...)
|
||||
|
||||
m.Server, err = pilosa.NewServer(serverOptions...)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
|
@ -403,6 +404,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
|
|||
|
||||
// Create new main with the same config.
|
||||
config := cluster[2].Command.Config
|
||||
config.Translation.MapSize = 100000
|
||||
// config.Bind = cluster[2].API.Node().URI.HostPort()
|
||||
|
||||
// this isn't necessary, but makes the test run way faster
|
||||
|
|
@ -476,6 +478,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
|
|||
|
||||
// Create new main with the same config.
|
||||
config := cluster[2].Command.Config
|
||||
config.Translation.MapSize = 100000
|
||||
// config.Bind = cluster[2].API.Node().URI.HostPort()
|
||||
|
||||
// this isn't necessary, but makes the test run way faster
|
||||
|
|
@ -496,6 +499,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
|
|||
// Create new main with the same config.
|
||||
config = cluster[1].Command.Config
|
||||
// config.Bind = cluster[1].API.Node().URI.HostPort()
|
||||
config.Translation.MapSize = 100000
|
||||
|
||||
// this isn't necessary, but makes the test run way faster
|
||||
config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port))
|
||||
|
|
@ -564,4 +568,128 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure program imports timestamps as UTC.
|
||||
func TestMain_ImportTimestamp(t *testing.T) {
|
||||
m := test.MustRunCommand()
|
||||
defer m.Close()
|
||||
|
||||
indexName := "i"
|
||||
fieldName := "f"
|
||||
|
||||
// Create index.
|
||||
if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create field.
|
||||
if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
data := pilosa.ImportRequest{
|
||||
Index: indexName,
|
||||
Field: fieldName,
|
||||
Shard: 0,
|
||||
RowIDs: []uint64{1, 2},
|
||||
ColumnIDs: []uint64{1, 2},
|
||||
Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00
|
||||
}
|
||||
|
||||
// Import data.
|
||||
if err := m.API.Import(context.Background(), &data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ensure the correct views were created.
|
||||
dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
exp := []string{
|
||||
"standard", "standard_2018", "standard_201801", "standard_20180101",
|
||||
"standard_2019", "standard_201912", "standard_20191231",
|
||||
}
|
||||
got := []string{}
|
||||
for _, f := range files {
|
||||
got = append(got, f.Name())
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(got, exp) {
|
||||
t.Fatalf("expected %v, but got %v", exp, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterQueriesAfterRestart(t *testing.T) {
|
||||
cluster := test.MustRunCluster(t, 3)
|
||||
defer cluster.Close()
|
||||
cmd1 := cluster[1]
|
||||
|
||||
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
|
||||
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
|
||||
|
||||
// build a query to set the first bit in 100 shards
|
||||
query := strings.Builder{}
|
||||
for i := 0; i < 100; i++ {
|
||||
query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth))
|
||||
}
|
||||
_, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
|
||||
Index: "testidx",
|
||||
Query: query.String(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("setting 100 bits in 100 shards: %v", err)
|
||||
}
|
||||
|
||||
results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
|
||||
Index: "testidx",
|
||||
Query: "Count(Row(testfield=0))",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("counting row: %v", err)
|
||||
}
|
||||
if results.Results[0].(uint64) != 100 {
|
||||
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
|
||||
}
|
||||
|
||||
err = cmd1.Command.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("closing node0: %v", err)
|
||||
}
|
||||
|
||||
// confirm that cluster stops accepting queries after one node closes
|
||||
if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
|
||||
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
|
||||
}
|
||||
|
||||
// Create new main with the same config.
|
||||
config := cmd1.Command.Config
|
||||
config.Bind = cmd1.API.Node().URI.HostPort()
|
||||
|
||||
// this isn't necessary, but makes the test run way faster
|
||||
config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port))
|
||||
cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr)
|
||||
cmd1.Command.Config = config
|
||||
err = cmd1.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("reopening node 0: %v", err)
|
||||
}
|
||||
|
||||
for cmd1.API.State() != pilosa.ClusterStateNormal {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
|
||||
Index: "testidx",
|
||||
Query: "Count(Row(testfield=0))",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("counting row: %v", err)
|
||||
}
|
||||
if results.Results[0].(uint64) != 100 {
|
||||
t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
|
||||
|
|
|
|||
|
|
@ -55,16 +55,21 @@ func newCommand(opts ...server.CommandOption) *Command {
|
|||
panic(err)
|
||||
}
|
||||
|
||||
// set aggressive close timeout by default to avoid hanging tests. This was
|
||||
// Set aggressive close timeout by default to avoid hanging tests. This was
|
||||
// a problem with PDK tests which used go-pilosa as well. We put it at the
|
||||
// beginning of the option slice so that it can be overridden by user-passed
|
||||
// options.
|
||||
opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...)
|
||||
// Also set TranslateFile MapSize to a smaller number so memory allocation
|
||||
// does not fail on 32-bit systems.
|
||||
opts = append([]server.CommandOption{
|
||||
server.OptCommandCloseTimeout(time.Millisecond * 2),
|
||||
}, opts...)
|
||||
m := &Command{commandOptions: opts}
|
||||
m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
|
||||
m.Config.DataDir = path
|
||||
m.Config.Bind = "http://localhost:0"
|
||||
m.Config.Cluster.Disabled = true
|
||||
m.Config.Translation.MapSize = 100000
|
||||
|
||||
if testing.Verbose() {
|
||||
m.Command.Stdout = os.Stdout
|
||||
|
|
|
|||
44
translate.go
44
translate.go
|
|
@ -5,7 +5,6 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
|
@ -17,6 +16,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -81,9 +81,29 @@ type TranslateFile struct {
|
|||
replicationRetryInterval time.Duration
|
||||
}
|
||||
|
||||
// TranslateFileOption is a functional option type for pilosa.TranslateFile
|
||||
type TranslateFileOption func(f *TranslateFile) error
|
||||
|
||||
func OptTranslateFileMapSize(mapSize int) TranslateFileOption {
|
||||
return func(f *TranslateFile) error {
|
||||
f.mapSize = mapSize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewTranslateFile returns a new instance of TranslateFile.
|
||||
func NewTranslateFile() *TranslateFile {
|
||||
return &TranslateFile{
|
||||
func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile {
|
||||
var defaultMapSize64 int64 = 10 * (1 << 30)
|
||||
var defaultMapSize int
|
||||
|
||||
if ^uint(0)>>32 > 0 {
|
||||
// 10GB default map size
|
||||
defaultMapSize = int(defaultMapSize64)
|
||||
} else {
|
||||
// Use 2GB default map size on 32-bit systems
|
||||
defaultMapSize = (1 << 31) - 1
|
||||
}
|
||||
f := &TranslateFile{
|
||||
writeNotify: make(chan struct{}),
|
||||
closing: make(chan struct{}),
|
||||
cols: make(map[string]*index),
|
||||
|
|
@ -96,25 +116,35 @@ func NewTranslateFile() *TranslateFile {
|
|||
|
||||
replicationRetryInterval: defaultReplicationRetryInterval,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
err := opt(f)
|
||||
if err != nil {
|
||||
// TODO (2.0): Change func signature to return error
|
||||
panic(errors.Wrap(err, "applying option"))
|
||||
}
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (s *TranslateFile) Open() (err error) {
|
||||
// Open writer & buffered writer.
|
||||
if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
|
||||
return err
|
||||
return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
|
||||
} else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil {
|
||||
return err
|
||||
return errors.Wrapf(err, "open file %s", s.Path)
|
||||
}
|
||||
s.w = bufio.NewWriter(s.file)
|
||||
|
||||
// Memory map data file.
|
||||
if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
|
||||
return err
|
||||
return errors.Wrapf(err, "creating Mmap (size: %d)", s.mapSize)
|
||||
}
|
||||
|
||||
// Replay the log.
|
||||
if err := s.replayEntries(); err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "replaying log entries")
|
||||
}
|
||||
|
||||
// Listen to primaryStoreEvents channel.
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
package pilosa
|
||||
|
||||
// defaultMapSize is the default size of mapped memory for the translate store.
|
||||
// It is passed as an int to syscall.Mmap and so must be < 2^31
|
||||
const defaultMapSize = (1 << 31) - 1 // 2GB
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
// +build !386
|
||||
|
||||
package pilosa
|
||||
|
||||
// defaultMapSize is the default size of mapped memory for the translate store.
|
||||
// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on
|
||||
// 64bit systems.
|
||||
const defaultMapSize = 10 * (1 << 30) // 10GB
|
||||
|
|
@ -367,7 +367,7 @@ func TestPrintTranslateFile(t *testing.T) {
|
|||
}
|
||||
f.Close()
|
||||
|
||||
s := pilosa.NewTranslateFile()
|
||||
s := pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))
|
||||
s.Path = f.Name()
|
||||
err = s.Open()
|
||||
if err != nil {
|
||||
|
|
@ -809,7 +809,7 @@ func NewTranslateFile() *TranslateFile {
|
|||
}
|
||||
f.Close()
|
||||
|
||||
s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()}
|
||||
s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))}
|
||||
s.Path = f.Name()
|
||||
return s
|
||||
}
|
||||
|
|
@ -847,7 +847,7 @@ func (s *TranslateFile) Reopen() error {
|
|||
}
|
||||
|
||||
s.lock.Lock()
|
||||
s.TranslateFile = pilosa.NewTranslateFile()
|
||||
s.TranslateFile = pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))
|
||||
s.lock.Unlock()
|
||||
s.Path = prev.Path
|
||||
s.SetPrimaryStore("restored-primary", prev.PrimaryTranslateStore)
|
||||
|
|
|
|||
2
view.go
2
view.go
|
|
@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
|
|||
frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
|
||||
if v.fieldType == FieldTypeMutex {
|
||||
frag.mutexVector = newRowsVector(frag)
|
||||
} else if v.fieldType == FieldTypeBool {
|
||||
frag.mutexVector = newBoolVector(frag)
|
||||
}
|
||||
return frag
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue