Merge branch 'master' into cluster-resize-merging

This commit is contained in:
Travis Turner 2018-02-16 11:13:40 -06:00
commit 6776fa81ef
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
38 changed files with 1624 additions and 192 deletions

16
Gopkg.lock generated
View file

@ -37,6 +37,12 @@
revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
version = "v1.3.1"
[[projects]]
name = "github.com/cespare/xxhash"
packages = ["."]
revision = "5c37fe3735342a2e0d01c87a907579987c8936cc"
version = "v1.0.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
@ -189,10 +195,16 @@
[[projects]]
name = "github.com/shirou/gopsutil"
packages = ["host","internal/common","mem","process"]
packages = ["cpu","host","internal/common","mem","net","process"]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
[[projects]]
branch = "master"
name = "github.com/shirou/w32"
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
name = "github.com/sony/gobreaker"
packages = ["."]
@ -268,6 +280,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56"
inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b"
solver-name = "gps-cdcl"
solver-version = 1

View file

@ -16,13 +16,15 @@ package pilosa
import (
"bytes"
"crypto/sha1"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
@ -242,7 +244,7 @@ func (s *AttrStore) Blocks() ([]AttrBlock, error) {
block := AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := sha1.New()
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
h.Write(k)
h.Write(v)

View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"fmt"
"reflect"
"testing"
@ -23,14 +24,34 @@ import (
// Ensure a bitmap can be merged
func TestBitmap_Merge(t *testing.T) {
bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth)
bm2 := pilosa.NewBitmap(3, 4, 5)
bm1.Merge(bm2)
if bm1.Count() != 7 {
t.Fatalf("Count after merge %d != 7\n", bm1.Count())
tests := []struct {
bm1 *pilosa.Bitmap
bm2 *pilosa.Bitmap
exp uint64
}{
{
bm1: pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth),
bm2: pilosa.NewBitmap(3, 4, 5),
exp: 7,
},
{
bm1: pilosa.NewBitmap(),
bm2: pilosa.NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004),
exp: 7,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) {
test.bm1.Merge(test.bm2)
if cnt := test.bm1.Count(); cnt != test.exp {
t.Fatalf("merged count %d is not %d", cnt, test.exp)
}
if length := len(test.bm1.Bits()); uint64(length) != test.exp {
t.Fatalf("merged length %d is not %d", length, test.exp)
}
})
}
}
// Ensure a bitmap can Xor'ed

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diagnostics_test
import (

View file

@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
The metrics configuration options are:
- Host to receive events
- Polling interval for runtime metrics
- Metric type (StatsD, Expvar).
- [Host](../configuration#metrics-host): specify host that receives metric events
- [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics
- [Service](../configuration#metrics-service): declare type StatsD or Expvar
##### Tags
StatsD Tags adhere to the DataDog format (key:value), and we tag the following:

View file

@ -13,13 +13,10 @@ nav = []
Returns the schema of all indexes in JSON.
Request:
```
``` request
curl -XGET localhost:10101/index
```
Response:
```
``` response
{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
```
@ -29,13 +26,10 @@ Response:
Returns the schema of the specified index in JSON.
Request:
```
``` request
curl -XGET localhost:10101/index/user
```
Response:
```
``` response
{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
```
@ -45,13 +39,10 @@ Response:
Creates an index with the given name.
Request:
```
``` request
curl -XPOST localhost:10101/index/user
```
Response:
```
``` response
{}
```
@ -61,13 +52,10 @@ Response:
Removes the given index.
Request:
```
``` request
curl -XDELETE localhost:10101/index/user
```
Response:
```
``` response
{}
```
@ -77,15 +65,12 @@ Response:
Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
Request:
```
``` request
curl localhost:10101/index/user/query \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
```
Response:
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
@ -95,14 +80,12 @@ The response doesn't include column attributes by default. To return them, set t
The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
Request:
```
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
-X POST \
-d 'Bitmap(frame="language", rowID=5)'
```
Response:
```
``` response
{
"results":[{"attrs":{},"bits":[100]}],
"columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}]
@ -134,21 +117,21 @@ Each individual `field` contains the following:
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
Request:
```
``` request
curl localhost:10101/index/user/frame/language \
-X POST \
-d '{"options": {"inverseEnabled": true}}'
```
``` response
{}
```
``` request
curl localhost:10101/index/repository/frame/stats \
-X POST \
-d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
```
Response:
```
``` response
{}
```
@ -158,13 +141,10 @@ Response:
Removes the given frame.
Request:
```
``` request
curl -XDELETE localhost:10101/index/user/frame/language
```
Response:
```
``` response
{}
```
@ -188,15 +168,12 @@ The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Va
* MDH: month, day and hour
* YMDH: year, month, day and hour
Request:
```
``` request
curl localhost:10101/index/user/frame/language/time-quantum \
-X POST \
-d '{"timeQuantum": "YM"}'
```
Response:
```
``` response
{}
```
@ -211,15 +188,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max`
* `min` (int): Minimum value allowed for this field.
* `max` (int): Maximum value allowed for this field.
Request:
```
``` request
curl localhost:10101/index/repository/frame/stats/field/pullrequests \
-X POST \
-d '{"type": "int", "min": 0, "max": 1000000}'
```
Response:
```
``` response
{}
```
@ -251,8 +225,7 @@ The `action` describes how the field value will be processed. Each `action` may
- `mapping`: Map the value to a RowID in the `valueMap`.
* `valueMap` (object): string and integer pairs used to map field values to RowID's.
Request:
```
``` request
curl localhost:10101/index/user/input-definition/stargazer-input \
-X POST \
-d '{
@ -284,9 +257,7 @@ curl localhost:10101/index/user/input-definition/stargazer-input \
]
}'
```
Response:
```
``` response
{}
```
@ -296,13 +267,10 @@ Response:
Returns the given input definition as JSON.
Request:
```
``` request
curl -XGET localhost:10101/index/user/input-definition/stargazer-input
```
Response:
```
``` response
{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]}
```
@ -312,13 +280,10 @@ Response:
Removes the given input definition.
Request:
```
``` request
curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
```
Response:
```
``` response
{}
```
@ -330,15 +295,12 @@ Processes the JSON payload using the given input definition.
The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition.
Request:
```
``` request
curl localhost:10101/index/user/input/stargazer-input \
-X POST \
-d '[{"language_id": "Go", "repo_id": 92274475}]'
```
Response:
```
``` response
{}
```
@ -348,13 +310,10 @@ Response:
Returns the hosts in the cluster.
Request:
```
``` request
curl -XGET localhost:10101/hosts
```
Response:
```
``` response
[{"host":":10101"}]
```
@ -364,13 +323,10 @@ Response:
Returns the version of the Pilosa server.
Request:
```
``` request
curl -XGET localhost:10101/version
```
Response:
```
``` response
{"version":"v0.6.0"}
```

View file

@ -219,7 +219,7 @@ Any flag that has a value that is a comma separated list on the command line bec
cpu-time = "30s"
```
##### Metric Service
* Description: Which stats service to use (StatsD or ExpVar).
* Description: Which stats service to use. Choose from [statsd, expvar].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd'
* Config:

View file

@ -31,11 +31,12 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest
```
Let's make sure Pilosa is running:
```
``` request
curl localhost:10101/status
```
Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}`
``` response
{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}
```
### Sample Project
@ -45,30 +46,41 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term
#### Create the Schema
Note:
The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
```
``` request
curl localhost:10101/schema
```
``` response
{"indexes":null}
```
Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first:
```
``` request
curl localhost:10101/index/repository -X POST
```
``` response
{}
```
Let's create the `stargazer` frame which has user IDs of stargazers as its rows:
```
``` request
curl localhost:10101/index/repository/frame/stargazer \
-X POST \
-d '{"options": {"timeQuantum": "YMD"}}'
```
``` response
{}
```
Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
Next up is the `language` frame, which will contain IDs for programming languages:
``` request
curl localhost:10101/index/repository/frame/language \
-X POST
```
curl localhost:10101/index/repository/frame/language -X POST
``` response
{}
```
#### Import Data From CSV Files
@ -107,46 +119,112 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d
</div>
Which repositories did user 14 star:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Bitmap(frame="stargazer", rowID=14)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
}
]
}
```
What are the top 5 languages in the sample data:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'TopN(frame="language", n=5)'
```
``` response
{
"results":[
[
{"id":5,"count":119},
{"id":1,"count":50},
{"id":4,"count":48},
{"id":9,"count":31},
{"id":13,"count":25}
]
]
}
```
Which repositories were starred by user 14 and 19:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))'
-d 'Intersect(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[2,3,362,396,416,461,464,466,470,486]
}
]
}
```
Which repositories were starred by user 14 or 19:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))'
-d 'Union(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
}
]
}
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))'
-d 'Intersect(
Bitmap(frame="stargazer", rowID=14),
Bitmap(frame="stargazer", rowID=19),
Bitmap(frame="language", rowID=1)
)'
```
``` response
{
"results":[
{
"attrs":{},
"bits":[2,362,416,461]
}
]
}
```
Set user 99999 as a stargazer for repository 77777:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)'
```
``` response
{"results":[true]}
```
### What's Next?

View file

@ -34,11 +34,14 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
```
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", columnID=10, rowID=1)'
```
``` response
{"results":[true]}
```
#### Arguments and Types

View file

@ -178,38 +178,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status
The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
All nodes should be in the `UP` state:
```
``` response
{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}}
```
#### Running Queries
Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d ''
```
``` response
{}
```
This will create index `sample-index` with default options. Let's create the frame now:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d ''
```
``` response
{}
```
We just created frame `sample-frame` with default options.
Let's run a `SetBit` query:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)'
```
``` response
{"results":[true]}
```
Confirm that the bit was indeed set:
```
``` request
curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
The same response should be returned when querying other nodes in the cluster:
```
``` request
curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)'
```
``` response
{"results":[{"attrs":{},"bits":[100]}]}
```
#### What's Next?
@ -223,20 +238,16 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
```
``` request
curl localhost:10101/index/patients \
-X POST
```
Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient.
```
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{"rangeEnabled": true}}'
``` response
{}
```
In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
```
``` request
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{
@ -248,17 +259,23 @@ curl localhost:10101/index/patients/frame/measurements \
]
}}'
```
``` response
{}
```
If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field).
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
This query sets the age, weight, and t-cell count for the patient with ID `1` in our system:
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)'
```
``` response
{"results":[null]}
```
In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
@ -282,39 +299,38 @@ pilosa import -i patients -f measurements --field age ages.csv
Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field.
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Range(frame="measurements", age > 40)'
```
You should get the following results:
```
``` response
{"results":[{"attrs":{},"bits":[2,6,9]}]}
```
You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(frame="measurements", field="age")'
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
```
``` response
{"results":[{"sum":377,"count":9}]}
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40.
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
```
``` response
{"results":[{"sum":191,"count":3}]}
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
### Storing Row and Column Attributes
@ -323,20 +339,26 @@ Notice in this case that the count is only `3` because of the `age > 40` filter
Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data.
First, create an index called `books` to use for this tutorial:
```
``` request
curl localhost:10101/index/books \
-X POST
```
``` response
{}
```
Next, create a frame in the `books` index called `members` which will represent library members who have read books.
```
``` request
curl localhost:10101/index/books/frame/members \
-X POST \
-d '{}'
```
``` response
{}
```
Now, let's add some books to our index.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960)
@ -345,9 +367,12 @@ curl localhost:10101/index/books/query \
SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(columnID=5, name="The Forever War", year=2008)'
```
``` response
{"results":[null,null,null,null,null]}
```
And add some members.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith")
@ -356,20 +381,22 @@ curl localhost:10101/index/books/query \
SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")'
```
``` response
{"results":[null,null,null,null,null]}
```
At this point we can query one of the `member` records by querying that row.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
You should get the following result set:
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
```
Now let's add some data to the matrix such that each pair represents a member who has read that book.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetBit(frame="members", rowID=10001, columnID=3)
@ -390,28 +417,28 @@ curl localhost:10101/index/books/query \
SetBit(frame="members", rowID=10005, columnID=4)
SetBit(frame="members", rowID=10005, columnID=5)'
```
``` response
{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
```
Now pull the record for `Sue Perkins` again.
```
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
```
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute.
```
``` response
{
"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}],
"columnAttrs":[
@ -421,15 +448,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt
]
}
```
The `book` attributes are included in the result set at the `columnAttrs` attribute.
Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members:
```
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))'
```
```
``` response
{
"results":[{"attrs":{},"bits":[4]}],
"columnAttrs":[

View file

@ -20,7 +20,6 @@ import (
"bytes"
"container/heap"
"context"
"crypto/sha1"
"encoding/binary"
"errors"
"fmt"
@ -36,6 +35,8 @@ import (
"time"
"unsafe"
"github.com/cespare/xxhash"
"math"
"github.com/gogo/protobuf/proto"
@ -1021,7 +1022,7 @@ type TopOptions struct {
// Checksum returns a checksum for the entire fragment.
// If two fragments have the same checksum then they have the same data.
func (f *Fragment) Checksum() []byte {
h := sha1.New()
h := xxhash.New()
for _, block := range f.Blocks() {
h.Write(block.Checksum)
}
@ -1665,7 +1666,7 @@ type blockHasher struct {
func newBlockHasher() blockHasher {
return blockHasher{
blockID: -1,
hash: sha1.New(),
hash: xxhash.New(),
}
}
func (h *blockHasher) Reset() {

View file

@ -1075,3 +1075,26 @@ func TestFragment_Snapshot_Run(t *testing.T) {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
func BenchmarkFragment_Snapshot(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
}
// Open the fragment specified by the path.
f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Close()
// Reset timer and execute benchmark.
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
err := f.Snapshot()
if err != nil {
b.Fatalf("unexpected count (reopen): %s", err)
}
}
}

View file

@ -377,7 +377,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 100 {
t.Fatalf("unexpected n: %d", n)
}
@ -465,7 +465,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 {
@ -526,7 +526,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
@ -599,7 +599,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs {
t.Fatalf("unexpected response type: %s", resp.Results[0].Type)
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (

View file

@ -840,6 +840,10 @@ func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) }
// Seek moves to the first value equal to or greater than `seek`.
func (itr *Iterator) Seek(seek uint64) {
// k should always be -1 unless we're seeking into a run container. Then the
// "if c.isRun" section will take care of it.
itr.k = -1
// Move to the correct container.
itr.i = search64(itr.bitmap.keys, highbits(seek))
if itr.i < 0 {
@ -2177,7 +2181,7 @@ func unionArrayArray(a, b *container) *container {
// unionArrayRun optimistically assumes that the result will be a run container,
// and converts to a bitmap or array container afterwards if necessary.
func unionArrayRun(a, b *container) *container {
if b.n == maxContainerVal {
if b.n == maxContainerVal+1 {
return b.clone()
}
output := &container{containerType: ContainerRun}
@ -2234,10 +2238,10 @@ func (c *container) runAppendInterval(v interval16) int {
}
func unionRunRun(a, b *container) *container {
if a.n == maxContainerVal {
if a.n == maxContainerVal+1 {
return a.clone()
}
if b.n == maxContainerVal {
if b.n == maxContainerVal+1 {
return b.clone()
}
na, nb := len(a.runs), len(b.runs)
@ -2268,9 +2272,12 @@ func unionRunRun(a, b *container) *container {
}
func unionBitmapRun(a, b *container) *container {
if b.n == maxContainerVal {
if b.n == maxContainerVal+1 {
return b.clone()
}
if a.n == maxContainerVal+1 {
return a.clone()
}
output := a.clone()
for j := 0; j < len(b.runs); j++ {
output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
@ -2285,7 +2292,7 @@ func (c *container) bitmapSetRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (64 - (j % 64))
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
xcnt := popcnt(X)
ycnt := popcnt(Y)
if x == y {
@ -2308,7 +2315,7 @@ func (c *container) bitmapXorRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (64 - (j % 64))
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
cnt := popcnt(c.bitmap[x])
c.bitmap[x] ^= (X & Y) //// flip
@ -2333,7 +2340,7 @@ func (c *container) bitmapZeroRange(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (64 - (j % 64))
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
c.n -= int(popcnt(c.bitmap[x] & (X & Y)))
c.bitmap[x] &= ^(X & Y)
@ -2469,10 +2476,18 @@ func differenceArrayRun(a, b *container) *container {
if i < len(a.array) {
// keep all array elements after end of runs
output.array = append(output.array, a.array[i:]...)
// TODO: consider handling container.n mutations in one place
// like we do with container.add().
output.n += len(a.array[i:])
// It's possible that output was converted from array to bitmap in output.add()
// so check container type before proceeding.
if output.containerType == ContainerArray {
output.array = append(output.array, a.array[i:]...)
// TODO: consider handling container.n mutations in one place
// like we do with container.add().
output.n += len(a.array[i:])
} else {
for _, v := range a.array[i:] {
output.add(v)
}
}
}
return output
}
@ -2500,6 +2515,8 @@ func differenceRunArray(a, b *container) *container {
bidx := 0
vb := b.array[bidx]
RUNLOOP:
for _, run := range a.runs {
start := run.start
for vb < run.start {
@ -2511,6 +2528,9 @@ func differenceRunArray(a, b *container) *container {
}
for vb >= run.start && vb <= run.last {
if vb == start {
if vb == 65535 { // overflow
break RUNLOOP
}
start++
bidx++
if bidx >= len(b.array) {
@ -2521,6 +2541,9 @@ func differenceRunArray(a, b *container) *container {
}
output.runs = append(output.runs, interval16{start: start, last: vb - 1})
output.n += int(vb - start)
if vb == 65535 { // overflow
break RUNLOOP
}
start = vb + 1
bidx++
if bidx >= len(b.array) {
@ -2772,7 +2795,9 @@ func xorArrayBitmap(a, b *container) *container {
}
}
if output.count() < ArrayMaxSize {
// It's possible that output was converted from bitmap to array in output.remove()
// so we only do this conversion if output is still a bitmap container.
if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize {
output.bitmapToArray()
}
@ -3187,7 +3212,7 @@ func xorRunRun(a, b *container) *container {
if nb == 0 {
return a.clone()
}
output := &container{}
output := &container{containerType: ContainerRun}
lastI, lastJ := -1, -1

View file

@ -0,0 +1,295 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
///////////////////////////////////////////////////////////////////////////
var containerWidth uint64 = 65536
////////////////// array
func arrayEmpty() []uint16 {
return make([]uint16, 0)
}
func arrayFull() []uint16 {
array := make([]uint16, containerWidth)
for i := 0; i < int(containerWidth); i++ {
array[i] = uint16(i)
}
return array
}
func arrayFirstBitSet() []uint16 {
array := make([]uint16, 0)
array = append(array, uint16(0))
return array
}
func arrayLastBitSet() []uint16 {
array := make([]uint16, 0)
array = append(array, uint16(65535))
return array
}
func arrayFirstBitUnset() []uint16 {
array := make([]uint16, containerWidth-1)
for i := 1; i < int(containerWidth); i++ {
array[i-1] = uint16(i)
}
return array
}
func arrayLastBitUnset() []uint16 {
array := make([]uint16, containerWidth-1)
for i := 0; i < int(containerWidth)-1; i++ {
array[i] = uint16(i)
}
return array
}
func arrayInnerBitsSet() []uint16 {
array := make([]uint16, containerWidth-2)
for i := 1; i < int(containerWidth)-1; i++ {
array[i-1] = uint16(i)
}
return array
}
func arrayOuterBitsSet() []uint16 {
return []uint16{0, 65535}
}
func arrayOddBitsSet() []uint16 {
array := make([]uint16, containerWidth/2)
for i := 0; i < int(containerWidth/2); i++ {
array[i] = uint16(2*i + 1)
}
return array
}
func arrayEvenBitsSet() []uint16 {
array := make([]uint16, containerWidth/2)
for i := 0; i < int(containerWidth/2); i++ {
array[i] = uint16(2 * i)
}
return array
}
////////////////// bitmap
func bitmapEmpty() []uint64 {
return make([]uint64, bitmapN)
}
func bitmapFull() []uint64 {
bitmap := make([]uint64, bitmapN)
for i := 0; i < bitmapN; i++ {
bitmap[i] = 0xFFFFFFFFFFFFFFFF
}
return bitmap
}
func bitmapFirstBitSet() []uint64 {
bitmap := make([]uint64, bitmapN)
bitmap[0] = 0x0000000000000001
return bitmap
}
func bitmapLastBitSet() []uint64 {
bitmap := make([]uint64, bitmapN)
bitmap[bitmapN-1] = 0x8000000000000000
return bitmap
}
func bitmapFirstBitUnset() []uint64 {
bitmap := bitmapFull()
bitmap[0] = 0xFFFFFFFFFFFFFFFE
return bitmap
}
func bitmapLastBitUnset() []uint64 {
bitmap := bitmapFull()
bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF
return bitmap
}
func bitmapInnerBitsSet() []uint64 {
bitmap := bitmapFull()
bitmap[0] = 0xFFFFFFFFFFFFFFFE
bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF
return bitmap
}
func bitmapOuterBitsSet() []uint64 {
bitmap := bitmapEmpty()
bitmap[0] = 0x0000000000000001
bitmap[bitmapN-1] = 0x8000000000000000
return bitmap
}
func bitmapOddBitsSet() []uint64 {
bitmap := make([]uint64, bitmapN)
for i := 0; i < bitmapN; i++ {
bitmap[i] = 0xAAAAAAAAAAAAAAAA
}
return bitmap
}
func bitmapEvenBitsSet() []uint64 {
bitmap := make([]uint64, bitmapN)
for i := 0; i < bitmapN; i++ {
bitmap[i] = 0x5555555555555555
}
return bitmap
}
////////////////// run
func runEmpty() []interval16 {
return make([]interval16, 0)
}
func runFull() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 0, last: 65535})
return run
}
func runFirstBitSet() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 0, last: 0})
return run
}
func runLastBitSet() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 65535, last: 65535})
return run
}
func runFirstBitUnset() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 1, last: 65535})
return run
}
func runLastBitUnset() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 0, last: 65534})
return run
}
func runInnerBitsSet() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 1, last: 65534})
return run
}
func runOuterBitsSet() []interval16 {
run := make([]interval16, 0)
run = append(run, interval16{start: 0, last: 0})
run = append(run, interval16{start: 65535, last: 65535})
return run
}
func runOddBitsSet() []interval16 {
run := make([]interval16, containerWidth/2)
for i := 0; i < int(containerWidth/2); i++ {
run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)}
}
return run
}
func runEvenBitsSet() []interval16 {
run := make([]interval16, containerWidth/2)
for i := 0; i < int(containerWidth/2); i++ {
run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)}
}
return run
}
///////////////////////////////////////////////////////////////////////////
type testOp struct {
f func(a, b *container) *container
x string
y string
exp string
}
func doContainer(containerType byte, data interface{}) *container {
c := &container{
containerType: containerType,
}
switch containerType {
case ContainerArray:
c.array = data.([]uint16)
case ContainerBitmap:
c.bitmap = data.([]uint64)
case ContainerRun:
c.runs = data.([]interval16)
}
c.n = c.count()
return c
}
func setupContainerTests() map[byte]map[string]*container {
cts := make(map[byte]map[string]*container)
// array containers
cts[ContainerArray] = map[string]*container{
"empty": doContainer(ContainerArray, arrayEmpty()),
"full": doContainer(ContainerArray, arrayFull()),
"firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(ContainerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()),
}
// bitmap containers
cts[ContainerBitmap] = map[string]*container{
"empty": doContainer(ContainerBitmap, bitmapEmpty()),
"full": doContainer(ContainerBitmap, bitmapFull()),
"firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()),
}
// run containers
cts[ContainerRun] = map[string]*container{
"empty": doContainer(ContainerRun, runEmpty()),
"full": doContainer(ContainerRun, runFull()),
"firstBitSet": doContainer(ContainerRun, runFirstBitSet()),
"lastBitSet": doContainer(ContainerRun, runLastBitSet()),
"firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(ContainerRun, runLastBitUnset()),
"innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(ContainerRun, runOddBitsSet()),
"evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()),
}
return cts
}

View file

@ -18,6 +18,8 @@ import (
"bytes"
"fmt"
"reflect"
"runtime"
"strings"
"testing"
)
@ -1479,6 +1481,16 @@ func TestDifferenceRunArray(t *testing.T) {
array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27},
exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}},
},
{
runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}},
array: []uint16{65533, 65534, 65535},
exp: []interval16{{start: 0, last: 20}},
},
{
runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}},
array: []uint16{37, 65535},
exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}},
},
}
for i, test := range tests {
a.runs = test.runs
@ -1583,14 +1595,69 @@ func TestDifferenceBitmapRun(t *testing.T) {
runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}},
exp: []uint64{0xFFFF0000FFFFFF0F},
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFBF},
runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}},
exp: []uint64{0x0000000000000000},
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFBF},
runs: []interval16{{start: 0, last: 5}},
exp: []uint64{0xFFFFFFFFFFFFFF80},
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
runs: []interval16{{start: 60, last: 63}},
exp: []uint64{0x0FFFFFFFFFFFFFFF},
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
runs: []interval16{{start: 60, last: 65}},
exp: []uint64{0x0FFFFFFFFFFFFFFF},
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}},
exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8},
},
{
bitmap: []uint64{0x0000000000000001},
runs: []interval16{{start: 0, last: 0}},
exp: []uint64{0x0000000000000000},
},
{
bitmap: []uint64{0x8000000000000000},
runs: []interval16{{start: 63, last: 63}},
exp: []uint64{0x0000000000000000},
},
{
bitmap: []uint64{0xC000000000000000, 0x0000000000000003},
runs: []interval16{{start: 63, last: 64}},
exp: []uint64{0x4000000000000000, 0x0000000000000002},
},
{
bitmap: []uint64{0x0000000000000000},
runs: []interval16{{start: 5, last: 7}},
exp: []uint64{0x0000000000000000},
},
{
bitmap: bitmapLastBitSet(),
runs: []interval16{{start: 65535, last: 65535}},
exp: bitmapEmpty(),
},
{
bitmap: bitmapFull(),
runs: []interval16{{start: 0, last: 65535}},
exp: bitmapEmpty(),
},
}
for i, test := range tests {
for i, v := range test.bitmap {
a.bitmap[i] = v
}
a.n = a.bitmapCountRange(0, 100)
a.n = a.bitmapCountRange(0, 65536)
b.runs = test.runs
b.n = b.runCountRange(0, 100)
b.n = b.runCountRange(0, 65536)
ret := differenceBitmapRun(a, b)
if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) {
t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)])
@ -1622,12 +1689,12 @@ func TestDifferenceBitmapArray(t *testing.T) {
exp: []uint16{8, 9, 11, 12, 13, 14, 15},
},
{
bitmap: bitmapOdds(),
bitmap: bitmapOddBitsSet(),
array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10},
exp: []uint16{9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63},
},
{
bitmap: bitmapOdds(),
bitmap: bitmapOddBitsSet(),
array: []uint16{63},
exp: []uint16{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61},
},
@ -2244,6 +2311,44 @@ func TestIteratorRuns(t *testing.T) {
}
}
func TestIteratorVarious(t *testing.T) {
tests := []struct {
bm *Bitmap
exp uint64
}{
{
bm: NewBitmap(3, 4, 5),
exp: 3,
},
{
bm: bitmapVariousContainers(),
exp: 61221,
},
{
bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004),
exp: 7,
},
}
for i, test := range tests {
test.bm.Optimize()
t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) {
if cnt := test.bm.Count(); cnt != test.exp {
t.Fatalf("merged count %d is not %d", cnt, test.exp)
}
iter := test.bm.Iterator()
bits := make([]uint64, 0, test.bm.Count())
for v, eof := iter.Next(); !eof; v, eof = iter.Next() {
bits = append(bits, v)
}
if length := len(bits); uint64(length) != test.exp {
t.Fatalf("length %d is not %d", length, test.exp)
}
})
}
}
func TestRunBinSearchContains(t *testing.T) {
tests := []struct {
runs []interval16
@ -2390,7 +2495,7 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) {
}
}
func TestSearc64(t *testing.T) {
func TestSearch64(t *testing.T) {
tests := []struct {
a []uint64
value uint64
@ -2505,12 +2610,12 @@ func TestIntersectArrayBitmap(t *testing.T) {
},
{
array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535},
bitmap: bitmapOdds(),
bitmap: bitmapOddBitsSet(),
exp: []uint16{1, 63, 543, 639, 65535},
},
{
array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535},
bitmap: bitmapEvens(),
bitmap: bitmapEvenBitsSet(),
exp: []uint16{0, 120, 12000, 65534},
},
}
@ -2532,18 +2637,606 @@ func TestIntersectArrayBitmap(t *testing.T) {
}
}
func bitmapOdds() []uint64 {
bitmap := make([]uint64, bitmapN)
for i := 0; i < bitmapN; i++ {
bitmap[i] = 0xAAAAAAAAAAAAAAAA
// rleCont returns a slice of numbers all in the range starting from
// container_width*num, and ending at container_width*(num+1)-1. If left is
// true, then the first 100 bits will be set, if mid is true, 100 bits in the
// middle will be set, if right is true, the last 100 bits will be set.
// sets 100 bits per true
func rleCont(num int, left, mid, right bool) []uint64 {
ret := make([]uint64, 0)
base := containerWidth * uint64(num)
if left {
for i := uint64(0); i < 100; i++ {
ret = append(ret, base+i)
}
}
return bitmap
if mid {
for i := containerWidth / 2; i < containerWidth/2+100; i++ {
ret = append(ret, base+i)
}
}
if right {
for i := containerWidth - 100; i < containerWidth; i++ {
ret = append(ret, base+i)
}
}
return ret
}
func bitmapEvens() []uint64 {
bitmap := make([]uint64, bitmapN)
for i := 0; i < bitmapN; i++ {
bitmap[i] = 0x5555555555555555
// sets 2 bits per true.
func arrCont(num int, left, mid, right bool) []uint64 {
ret := make([]uint64, 0)
base := containerWidth * uint64(num)
if left {
ret = append(ret, base+0, base+2)
}
if mid {
half := containerWidth / 2
ret = append(ret, base+half, base+half+2)
}
if right {
ret = append(ret, base+containerWidth-3, base+containerWidth-1)
}
return ret
}
// sets 6667 bits per true.
func bitCont(num int, left, mid, right bool) []uint64 {
ret := make([]uint64, 0)
base := containerWidth * uint64(num)
if left {
for i := uint64(0); i < 20001; i += 3 {
ret = append(ret, base+i)
}
}
if mid {
for i := uint64(21000); i < 41001; i += 3 {
ret = append(ret, base+i)
}
}
if right {
for i := uint64(45537); i <= 65535; i += 3 {
ret = append(ret, base+i)
}
}
return ret
}
func bitmapVariousContainers() *Bitmap {
bits := make([]uint64, 0)
bits = append(bits, rleCont(0, true, true, true)...)
bits = append(bits, rleCont(1, true, true, true)...)
bits = append(bits, arrCont(2, true, true, true)...)
bits = append(bits, arrCont(3, true, true, true)...)
bits = append(bits, bitCont(4, true, true, true)...)
bits = append(bits, bitCont(5, true, true, true)...)
bits = append(bits, rleCont(6, true, true, true)...)
bits = append(bits, bitCont(7, true, true, true)...)
bits = append(bits, arrCont(8, true, true, true)...)
bits = append(bits, rleCont(9, true, true, true)...)
bm := NewBitmap(bits...)
bm.Optimize()
return bm
}
///////////////////////////////////////////////////////////////////////////
func getFunctionName(i interface{}) string {
x := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
y := strings.Split(x, ".")
y = y[len(y)-1:]
return y[0]
}
func TestContainerCombinations(t *testing.T) {
cts := setupContainerTests()
containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun}
// map used for a more descriptive print
cm := map[byte]string{
ContainerArray: "array",
ContainerBitmap: "bitmap",
ContainerRun: "run",
}
testOps := []testOp{
// intersect
{intersect, "empty", "empty", "empty"},
{intersect, "empty", "full", "empty"},
{intersect, "empty", "firstBitSet", "empty"},
{intersect, "empty", "lastBitSet", "empty"},
{intersect, "empty", "firstBitUnset", "empty"},
{intersect, "empty", "lastBitUnset", "empty"},
{intersect, "empty", "innerBitsSet", "empty"},
{intersect, "empty", "outerBitsSet", "empty"},
{intersect, "empty", "oddBitsSet", "empty"},
{intersect, "empty", "evenBitsSet", "empty"},
//
{intersect, "full", "empty", "empty"},
{intersect, "full", "full", "full"},
{intersect, "full", "firstBitSet", "firstBitSet"},
{intersect, "full", "lastBitSet", "lastBitSet"},
{intersect, "full", "firstBitUnset", "firstBitUnset"},
{intersect, "full", "lastBitUnset", "lastBitUnset"},
{intersect, "full", "innerBitsSet", "innerBitsSet"},
{intersect, "full", "outerBitsSet", "outerBitsSet"},
{intersect, "full", "oddBitsSet", "oddBitsSet"},
{intersect, "full", "evenBitsSet", "evenBitsSet"},
//
{intersect, "firstBitSet", "empty", "empty"},
{intersect, "firstBitSet", "full", "firstBitSet"},
{intersect, "firstBitSet", "firstBitSet", "firstBitSet"},
{intersect, "firstBitSet", "lastBitSet", "empty"},
{intersect, "firstBitSet", "firstBitUnset", "empty"},
{intersect, "firstBitSet", "lastBitUnset", "firstBitSet"},
{intersect, "firstBitSet", "innerBitsSet", "empty"},
{intersect, "firstBitSet", "outerBitsSet", "firstBitSet"},
{intersect, "firstBitSet", "oddBitsSet", "empty"},
{intersect, "firstBitSet", "evenBitsSet", "firstBitSet"},
//
{intersect, "lastBitSet", "empty", "empty"},
{intersect, "lastBitSet", "full", "lastBitSet"},
{intersect, "lastBitSet", "firstBitSet", "empty"},
{intersect, "lastBitSet", "lastBitSet", "lastBitSet"},
{intersect, "lastBitSet", "firstBitUnset", "lastBitSet"},
{intersect, "lastBitSet", "lastBitUnset", "empty"},
{intersect, "lastBitSet", "innerBitsSet", "empty"},
{intersect, "lastBitSet", "outerBitsSet", "lastBitSet"},
{intersect, "lastBitSet", "oddBitsSet", "lastBitSet"},
{intersect, "lastBitSet", "evenBitsSet", "empty"},
//
{intersect, "firstBitUnset", "empty", "empty"},
{intersect, "firstBitUnset", "full", "firstBitUnset"},
{intersect, "firstBitUnset", "firstBitSet", "empty"},
{intersect, "firstBitUnset", "lastBitSet", "lastBitSet"},
{intersect, "firstBitUnset", "firstBitUnset", "firstBitUnset"},
{intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"},
{intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"},
{intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"},
{intersect, "firstBitUnset", "oddBitsSet", "oddBitsSet"},
//{intersect, "firstBitUnset", "evenBitsSet", ""},
//
{intersect, "lastBitUnset", "empty", "empty"},
{intersect, "lastBitUnset", "full", "lastBitUnset"},
{intersect, "lastBitUnset", "firstBitSet", "firstBitSet"},
{intersect, "lastBitUnset", "lastBitSet", "empty"},
{intersect, "lastBitUnset", "firstBitUnset", "innerBitsSet"},
{intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"},
{intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"},
{intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"},
//{intersect, "lastBitUnset", "oddBitsSet", ""},
{intersect, "lastBitUnset", "evenBitsSet", "evenBitsSet"},
//
{intersect, "innerBitsSet", "empty", "empty"},
{intersect, "innerBitsSet", "full", "innerBitsSet"},
{intersect, "innerBitsSet", "firstBitSet", "empty"},
{intersect, "innerBitsSet", "lastBitSet", "empty"},
{intersect, "innerBitsSet", "firstBitUnset", "innerBitsSet"},
{intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"},
{intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"},
{intersect, "innerBitsSet", "outerBitsSet", "empty"},
//{intersect, "innerBitsSet", "oddBitsSet", ""},
//{intersect, "innerBitsSet", "evenBitsSet", ""},
//
{intersect, "outerBitsSet", "empty", "empty"},
{intersect, "outerBitsSet", "full", "outerBitsSet"},
{intersect, "outerBitsSet", "firstBitSet", "firstBitSet"},
{intersect, "outerBitsSet", "lastBitSet", "lastBitSet"},
{intersect, "outerBitsSet", "firstBitUnset", "lastBitSet"},
{intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"},
{intersect, "outerBitsSet", "innerBitsSet", "empty"},
{intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"},
{intersect, "outerBitsSet", "oddBitsSet", "lastBitSet"},
{intersect, "outerBitsSet", "evenBitsSet", "firstBitSet"},
//
{intersect, "oddBitsSet", "empty", "empty"},
{intersect, "oddBitsSet", "full", "oddBitsSet"},
{intersect, "oddBitsSet", "firstBitSet", "empty"},
{intersect, "oddBitsSet", "lastBitSet", "lastBitSet"},
{intersect, "oddBitsSet", "firstBitUnset", "oddBitsSet"},
//{intersect, "oddBitsSet", "lastBitUnset", ""},
//{intersect, "oddBitsSet", "innerBitsSet", ""},
{intersect, "oddBitsSet", "outerBitsSet", "lastBitSet"},
{intersect, "oddBitsSet", "oddBitsSet", "oddBitsSet"},
{intersect, "oddBitsSet", "evenBitsSet", "empty"},
//
{intersect, "evenBitsSet", "empty", "empty"},
{intersect, "evenBitsSet", "full", "evenBitsSet"},
{intersect, "evenBitsSet", "firstBitSet", "firstBitSet"},
{intersect, "evenBitsSet", "lastBitSet", "empty"},
//{intersect, "evenBitsSet", "firstBitUnset", ""},
{intersect, "evenBitsSet", "lastBitUnset", "evenBitsSet"},
//{intersect, "evenBitsSet", "innerBitsSet", ""},
{intersect, "evenBitsSet", "outerBitsSet", "firstBitSet"},
{intersect, "evenBitsSet", "oddBitsSet", "empty"},
{intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"},
// union
{union, "empty", "empty", "empty"},
{union, "empty", "full", "full"},
{union, "empty", "firstBitSet", "firstBitSet"},
{union, "empty", "lastBitSet", "lastBitSet"},
{union, "empty", "firstBitUnset", "firstBitUnset"},
{union, "empty", "lastBitUnset", "lastBitUnset"},
{union, "empty", "innerBitsSet", "innerBitsSet"},
{union, "empty", "outerBitsSet", "outerBitsSet"},
{union, "empty", "oddBitsSet", "oddBitsSet"},
{union, "empty", "evenBitsSet", "evenBitsSet"},
//
{union, "full", "empty", "full"},
{union, "full", "full", "full"},
{union, "full", "firstBitSet", "full"},
{union, "full", "lastBitSet", "full"},
{union, "full", "firstBitUnset", "full"},
{union, "full", "lastBitUnset", "full"},
{union, "full", "innerBitsSet", "full"},
{union, "full", "outerBitsSet", "full"},
{union, "full", "oddBitsSet", "full"},
{union, "full", "evenBitsSet", "full"},
//
{union, "firstBitSet", "empty", "firstBitSet"},
{union, "firstBitSet", "full", "full"},
{union, "firstBitSet", "firstBitSet", "firstBitSet"},
{union, "firstBitSet", "lastBitSet", "outerBitsSet"},
{union, "firstBitSet", "firstBitUnset", "full"},
{union, "firstBitSet", "lastBitUnset", "lastBitUnset"},
{union, "firstBitSet", "innerBitsSet", "lastBitUnset"},
{union, "firstBitSet", "outerBitsSet", "outerBitsSet"},
//{union, "firstBitSet", "oddBitsSet", ""},
{union, "firstBitSet", "evenBitsSet", "evenBitsSet"},
//
{union, "lastBitSet", "empty", "lastBitSet"},
{union, "lastBitSet", "full", "full"},
{union, "lastBitSet", "firstBitSet", "outerBitsSet"},
{union, "lastBitSet", "lastBitSet", "lastBitSet"},
{union, "lastBitSet", "firstBitUnset", "firstBitUnset"},
{union, "lastBitSet", "lastBitUnset", "full"},
{union, "lastBitSet", "innerBitsSet", "firstBitUnset"},
{union, "lastBitSet", "outerBitsSet", "outerBitsSet"},
{union, "lastBitSet", "oddBitsSet", "oddBitsSet"},
//{union, "lastBitSet", "evenBitsSet", ""},
//
{union, "firstBitUnset", "empty", "firstBitUnset"},
{union, "firstBitUnset", "full", "full"},
{union, "firstBitUnset", "firstBitSet", "full"},
{union, "firstBitUnset", "lastBitSet", "firstBitUnset"},
{union, "firstBitUnset", "firstBitUnset", "firstBitUnset"},
{union, "firstBitUnset", "lastBitUnset", "full"},
{union, "firstBitUnset", "innerBitsSet", "firstBitUnset"},
{union, "firstBitUnset", "outerBitsSet", "full"},
{union, "firstBitUnset", "oddBitsSet", "firstBitUnset"},
{union, "firstBitUnset", "evenBitsSet", "full"},
//
{union, "lastBitUnset", "empty", "lastBitUnset"},
{union, "lastBitUnset", "full", "full"},
{union, "lastBitUnset", "firstBitSet", "lastBitUnset"},
{union, "lastBitUnset", "lastBitSet", "full"},
{union, "lastBitUnset", "firstBitUnset", "full"},
{union, "lastBitUnset", "lastBitUnset", "lastBitUnset"},
{union, "lastBitUnset", "innerBitsSet", "lastBitUnset"},
{union, "lastBitUnset", "outerBitsSet", "full"},
{union, "lastBitUnset", "oddBitsSet", "full"},
{union, "lastBitUnset", "evenBitsSet", "lastBitUnset"},
//
{union, "innerBitsSet", "empty", "innerBitsSet"},
{union, "innerBitsSet", "full", "full"},
{union, "innerBitsSet", "firstBitSet", "lastBitUnset"},
{union, "innerBitsSet", "lastBitSet", "firstBitUnset"},
{union, "innerBitsSet", "firstBitUnset", "firstBitUnset"},
{union, "innerBitsSet", "lastBitUnset", "lastBitUnset"},
{union, "innerBitsSet", "innerBitsSet", "innerBitsSet"},
{union, "innerBitsSet", "outerBitsSet", "full"},
{union, "innerBitsSet", "oddBitsSet", "firstBitUnset"},
{union, "innerBitsSet", "evenBitsSet", "lastBitUnset"},
//
{union, "outerBitsSet", "empty", "outerBitsSet"},
{union, "outerBitsSet", "full", "full"},
{union, "outerBitsSet", "firstBitSet", "outerBitsSet"},
{union, "outerBitsSet", "lastBitSet", "outerBitsSet"},
{union, "outerBitsSet", "firstBitUnset", "full"},
{union, "outerBitsSet", "lastBitUnset", "full"},
{union, "outerBitsSet", "innerBitsSet", "full"},
{union, "outerBitsSet", "outerBitsSet", "outerBitsSet"},
//{union, "outerBitsSet", "oddBitsSet", ""},
//{union, "outerBitsSet", "evenBitsSet", ""},
//
{union, "oddBitsSet", "empty", "oddBitsSet"},
{union, "oddBitsSet", "full", "full"},
//{union, "oddBitsSet", "firstBitSet", ""},
{union, "oddBitsSet", "lastBitSet", "oddBitsSet"},
{union, "oddBitsSet", "firstBitUnset", "firstBitUnset"},
{union, "oddBitsSet", "lastBitUnset", "full"},
{union, "oddBitsSet", "innerBitsSet", "firstBitUnset"},
//{union, "oddBitsSet", "outerBitsSet", ""},
{union, "oddBitsSet", "oddBitsSet", "oddBitsSet"},
{union, "oddBitsSet", "evenBitsSet", "full"},
//
{union, "evenBitsSet", "empty", "evenBitsSet"},
{union, "evenBitsSet", "full", "full"},
{union, "evenBitsSet", "firstBitSet", "evenBitsSet"},
//{union, "evenBitsSet", "lastBitSet", ""},
{union, "evenBitsSet", "firstBitUnset", "full"},
{union, "evenBitsSet", "lastBitUnset", "lastBitUnset"},
{union, "evenBitsSet", "innerBitsSet", "lastBitUnset"},
//{union, "evenBitsSet", "outerBitsSet", ""},
{union, "evenBitsSet", "oddBitsSet", "full"},
{union, "evenBitsSet", "evenBitsSet", "evenBitsSet"},
// difference
{difference, "empty", "empty", "empty"},
{difference, "empty", "full", "empty"},
{difference, "empty", "firstBitSet", "empty"},
{difference, "empty", "lastBitSet", "empty"},
{difference, "empty", "firstBitUnset", "empty"},
{difference, "empty", "lastBitUnset", "empty"},
{difference, "empty", "innerBitsSet", "empty"},
{difference, "empty", "outerBitsSet", "empty"},
{difference, "empty", "oddBitsSet", "empty"},
{difference, "empty", "evenBitsSet", "empty"},
//
{difference, "full", "empty", "full"},
{difference, "full", "full", "empty"},
{difference, "full", "firstBitSet", "firstBitUnset"},
{difference, "full", "lastBitSet", "lastBitUnset"},
{difference, "full", "firstBitUnset", "firstBitSet"},
{difference, "full", "lastBitUnset", "lastBitSet"},
{difference, "full", "innerBitsSet", "outerBitsSet"},
{difference, "full", "outerBitsSet", "innerBitsSet"},
{difference, "full", "oddBitsSet", "evenBitsSet"},
{difference, "full", "evenBitsSet", "oddBitsSet"},
//
{difference, "firstBitSet", "empty", "firstBitSet"},
{difference, "firstBitSet", "full", "empty"},
{difference, "firstBitSet", "firstBitSet", "empty"},
{difference, "firstBitSet", "lastBitSet", "firstBitSet"},
{difference, "firstBitSet", "firstBitUnset", "firstBitSet"},
{difference, "firstBitSet", "lastBitUnset", "empty"},
{difference, "firstBitSet", "innerBitsSet", "firstBitSet"},
{difference, "firstBitSet", "outerBitsSet", "empty"},
{difference, "firstBitSet", "oddBitsSet", "firstBitSet"},
{difference, "firstBitSet", "evenBitsSet", "empty"},
//
{difference, "lastBitSet", "empty", "lastBitSet"},
{difference, "lastBitSet", "full", "empty"},
{difference, "lastBitSet", "firstBitSet", "lastBitSet"},
{difference, "lastBitSet", "lastBitSet", "empty"},
{difference, "lastBitSet", "firstBitUnset", "empty"},
{difference, "lastBitSet", "lastBitUnset", "lastBitSet"},
{difference, "lastBitSet", "innerBitsSet", "lastBitSet"},
{difference, "lastBitSet", "outerBitsSet", "empty"},
{difference, "lastBitSet", "oddBitsSet", "empty"},
{difference, "lastBitSet", "evenBitsSet", "lastBitSet"},
//
{difference, "firstBitUnset", "empty", "firstBitUnset"},
{difference, "firstBitUnset", "full", "empty"},
{difference, "firstBitUnset", "firstBitSet", "firstBitUnset"},
{difference, "firstBitUnset", "lastBitSet", "innerBitsSet"},
{difference, "firstBitUnset", "firstBitUnset", "empty"},
{difference, "firstBitUnset", "lastBitUnset", "lastBitSet"},
{difference, "firstBitUnset", "innerBitsSet", "lastBitSet"},
{difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"},
//{difference, "firstBitUnset", "oddBitsSet", ""},
{difference, "firstBitUnset", "evenBitsSet", "oddBitsSet"},
//
{difference, "lastBitUnset", "empty", "lastBitUnset"},
{difference, "lastBitUnset", "full", "empty"},
{difference, "lastBitUnset", "firstBitSet", "innerBitsSet"},
{difference, "lastBitUnset", "lastBitSet", "lastBitUnset"},
{difference, "lastBitUnset", "firstBitUnset", "firstBitSet"},
{difference, "lastBitUnset", "lastBitUnset", "empty"},
{difference, "lastBitUnset", "innerBitsSet", "firstBitSet"},
{difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"},
{difference, "lastBitUnset", "oddBitsSet", "evenBitsSet"},
//{difference, "lastBitUnset", "evenBitsSet", ""},
//
{difference, "innerBitsSet", "empty", "innerBitsSet"},
{difference, "innerBitsSet", "full", "empty"},
{difference, "innerBitsSet", "firstBitSet", "innerBitsSet"},
{difference, "innerBitsSet", "lastBitSet", "innerBitsSet"},
{difference, "innerBitsSet", "firstBitUnset", "empty"},
{difference, "innerBitsSet", "lastBitUnset", "empty"},
{difference, "innerBitsSet", "innerBitsSet", "empty"},
{difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"},
//{difference, "innerBitsSet", "oddBitsSet", ""},
//{difference, "innerBitsSet", "evenBitsSet", ""},
//
{difference, "outerBitsSet", "empty", "outerBitsSet"},
{difference, "outerBitsSet", "full", "empty"},
{difference, "outerBitsSet", "firstBitSet", "lastBitSet"},
{difference, "outerBitsSet", "lastBitSet", "firstBitSet"},
{difference, "outerBitsSet", "firstBitUnset", "firstBitSet"},
{difference, "outerBitsSet", "lastBitUnset", "lastBitSet"},
{difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"},
{difference, "outerBitsSet", "outerBitsSet", "empty"},
{difference, "outerBitsSet", "oddBitsSet", "firstBitSet"},
{difference, "outerBitsSet", "evenBitsSet", "lastBitSet"},
//
{difference, "oddBitsSet", "empty", "oddBitsSet"},
{difference, "oddBitsSet", "full", "empty"},
{difference, "oddBitsSet", "firstBitSet", "oddBitsSet"},
//{difference, "oddBitsSet", "lastBitSet", ""},
{difference, "oddBitsSet", "firstBitUnset", "empty"},
{difference, "oddBitsSet", "lastBitUnset", "lastBitSet"},
{difference, "oddBitsSet", "innerBitsSet", "lastBitSet"},
//{difference, "oddBitsSet", "outerBitsSet", ""},
{difference, "oddBitsSet", "oddBitsSet", "empty"},
{difference, "oddBitsSet", "evenBitsSet", "oddBitsSet"},
//
{difference, "evenBitsSet", "empty", "evenBitsSet"},
{difference, "evenBitsSet", "full", "empty"},
//{difference, "evenBitsSet", "firstBitSet", ""},
{difference, "evenBitsSet", "lastBitSet", "evenBitsSet"},
{difference, "evenBitsSet", "firstBitUnset", "firstBitSet"},
{difference, "evenBitsSet", "lastBitUnset", "empty"},
{difference, "evenBitsSet", "innerBitsSet", "firstBitSet"},
//{difference, "evenBitsSet", "outerBitsSet", ""},
{difference, "evenBitsSet", "oddBitsSet", "evenBitsSet"},
{difference, "evenBitsSet", "evenBitsSet", "empty"},
// xor
{xor, "empty", "empty", "empty"},
{xor, "empty", "full", "full"},
{xor, "empty", "firstBitSet", "firstBitSet"},
{xor, "empty", "lastBitSet", "lastBitSet"},
{xor, "empty", "firstBitUnset", "firstBitUnset"},
{xor, "empty", "lastBitUnset", "lastBitUnset"},
{xor, "empty", "innerBitsSet", "innerBitsSet"},
{xor, "empty", "outerBitsSet", "outerBitsSet"},
{xor, "empty", "oddBitsSet", "oddBitsSet"},
{xor, "empty", "evenBitsSet", "evenBitsSet"},
//
{xor, "full", "empty", "full"},
{xor, "full", "full", "empty"},
{xor, "full", "firstBitSet", "firstBitUnset"},
{xor, "full", "lastBitSet", "lastBitUnset"},
{xor, "full", "firstBitUnset", "firstBitSet"},
{xor, "full", "lastBitUnset", "lastBitSet"},
{xor, "full", "innerBitsSet", "outerBitsSet"},
{xor, "full", "outerBitsSet", "innerBitsSet"},
{xor, "full", "oddBitsSet", "evenBitsSet"},
{xor, "full", "evenBitsSet", "oddBitsSet"},
//
{xor, "firstBitSet", "empty", "firstBitSet"},
{xor, "firstBitSet", "full", "firstBitUnset"},
{xor, "firstBitSet", "firstBitSet", "empty"},
{xor, "firstBitSet", "lastBitSet", "outerBitsSet"},
{xor, "firstBitSet", "firstBitUnset", "full"},
{xor, "firstBitSet", "lastBitUnset", "innerBitsSet"},
{xor, "firstBitSet", "innerBitsSet", "lastBitUnset"},
{xor, "firstBitSet", "outerBitsSet", "lastBitSet"},
//{xor, "firstBitSet", "oddBitsSet", ""},
//{xor, "firstBitSet", "evenBitsSet", ""},
//
{xor, "lastBitSet", "empty", "lastBitSet"},
{xor, "lastBitSet", "full", "lastBitUnset"},
{xor, "lastBitSet", "firstBitSet", "outerBitsSet"},
{xor, "lastBitSet", "lastBitSet", "empty"},
{xor, "lastBitSet", "firstBitUnset", "innerBitsSet"},
{xor, "lastBitSet", "lastBitUnset", "full"},
{xor, "lastBitSet", "innerBitsSet", "firstBitUnset"},
{xor, "lastBitSet", "outerBitsSet", "firstBitSet"},
//{xor, "lastBitSet", "oddBitsSet", ""},
//{xor, "lastBitSet", "evenBitsSet", ""},
//
{xor, "firstBitUnset", "empty", "firstBitUnset"},
{xor, "firstBitUnset", "full", "firstBitSet"},
{xor, "firstBitUnset", "firstBitSet", "full"},
{xor, "firstBitUnset", "lastBitSet", "innerBitsSet"},
{xor, "firstBitUnset", "firstBitUnset", "empty"},
{xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"},
{xor, "firstBitUnset", "innerBitsSet", "lastBitSet"},
{xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"},
//{xor, "firstBitUnset", "oddBitsSet", ""},
//{xor, "firstBitUnset", "evenBitsSet", ""},
//
{xor, "lastBitUnset", "empty", "lastBitUnset"},
{xor, "lastBitUnset", "full", "lastBitSet"},
{xor, "lastBitUnset", "firstBitSet", "innerBitsSet"},
{xor, "lastBitUnset", "lastBitSet", "full"},
{xor, "lastBitUnset", "firstBitUnset", "outerBitsSet"},
{xor, "lastBitUnset", "lastBitUnset", "empty"},
{xor, "lastBitUnset", "innerBitsSet", "firstBitSet"},
{xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"},
//{xor, "lastBitUnset", "oddBitsSet", ""},
//{xor, "lastBitUnset", "evenBitsSet", ""},
//
{xor, "innerBitsSet", "empty", "innerBitsSet"},
{xor, "innerBitsSet", "full", "outerBitsSet"},
{xor, "innerBitsSet", "firstBitSet", "lastBitUnset"},
{xor, "innerBitsSet", "lastBitSet", "firstBitUnset"},
{xor, "innerBitsSet", "firstBitUnset", "lastBitSet"},
{xor, "innerBitsSet", "lastBitUnset", "firstBitSet"},
{xor, "innerBitsSet", "innerBitsSet", "empty"},
{xor, "innerBitsSet", "outerBitsSet", "full"},
//{xor, "innerBitsSet", "oddBitsSet", ""},
//{xor, "innerBitsSet", "evenBitsSet", ""},
//
{xor, "outerBitsSet", "empty", "outerBitsSet"},
{xor, "outerBitsSet", "full", "innerBitsSet"},
{xor, "outerBitsSet", "firstBitSet", "lastBitSet"},
{xor, "outerBitsSet", "lastBitSet", "firstBitSet"},
{xor, "outerBitsSet", "firstBitUnset", "lastBitUnset"},
{xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"},
{xor, "outerBitsSet", "innerBitsSet", "full"},
{xor, "outerBitsSet", "outerBitsSet", "empty"},
//{xor, "outerBitsSet", "oddBitsSet", ""},
//{xor, "outerBitsSet", "evenBitsSet", ""},
//
{xor, "oddBitsSet", "empty", "oddBitsSet"},
{xor, "oddBitsSet", "full", "evenBitsSet"},
//{xor, "oddBitsSet", "firstBitSet", ""},
//{xor, "oddBitsSet", "lastBitSet", ""},
//{xor, "oddBitsSet", "firstBitUnset", ""},
//{xor, "oddBitsSet", "lastBitUnset", ""},
//{xor, "oddBitsSet", "innerBitsSet", ""},
//{xor, "oddBitsSet", "outerBitsSet", ""},
{xor, "oddBitsSet", "oddBitsSet", "empty"},
{xor, "oddBitsSet", "evenBitsSet", "full"},
//
{xor, "evenBitsSet", "empty", "evenBitsSet"},
{xor, "evenBitsSet", "full", "oddBitsSet"},
//{xor, "evenBitsSet", "firstBitSet", ""},
//{xor, "evenBitsSet", "lastBitSet", ""},
//{xor, "evenBitsSet", "firstBitUnset", ""},
//{xor, "evenBitsSet", "lastBitUnset", ""},
//{xor, "evenBitsSet", "innerBitsSet", ""},
//{xor, "evenBitsSet", "outerBitsSet", ""},
{xor, "evenBitsSet", "oddBitsSet", "full"},
{xor, "evenBitsSet", "evenBitsSet", "empty"},
}
for _, testOp := range testOps {
for _, x := range containerTypes {
for _, y := range containerTypes {
desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y)
ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y])
exp := testOp.exp
// Convert to all container types and check result.
for _, ct := range containerTypes {
clone := ret.clone()
if ct == ContainerArray {
if clone.isBitmap() {
clone.bitmapToArray()
} else if clone.isRun() {
clone.runToArray()
}
if clone.n != cts[ct][exp].n {
t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n)
}
// Because xorRunRun resulting in an empty container returns an array container with a
// nil slice array, then we need to check len() on array first (look for 0).
if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) {
t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array)
}
} else if ct == ContainerBitmap {
if clone.isArray() {
clone.arrayToBitmap()
} else if clone.isRun() {
clone.runToBitmap()
}
if clone.n != cts[ct][exp].n {
t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n)
}
if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) {
t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap)
}
} else if ct == ContainerRun {
if clone.isArray() {
clone.arrayToRun()
} else if clone.isBitmap() {
clone.bitmapToRun()
}
if clone.n != cts[ct][exp].n {
t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n)
}
if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) {
t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs)
}
}
}
}
}
}
return bitmap
}

View file

@ -15,6 +15,7 @@
package pilosa
import (
"context"
"crypto/tls"
"errors"
"fmt"

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build !release
//
// This file sets defaults to be overridden by release.go

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build release
//
// This file sets release-specific variables.

View file

@ -16,6 +16,7 @@
// itself. The purpose of this package is to define an easily tested Command
// object which handles interpreting configuration and setting up all the
// objects that Pilosa needs.
package server
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test_test
import (

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import "flag"

View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
var Version = "v0.0.0"