diff --git a/Gopkg.lock b/Gopkg.lock index 7837cadff..a8071c915 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,12 @@ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" +[[projects]] + name = "github.com/cespare/xxhash" + packages = ["."] + revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" + version = "v1.0.0" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -189,10 +195,16 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["host","internal/common","mem","process"] + packages = ["cpu","host","internal/common","mem","net","process"] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -268,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" + inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/attr.go b/attr.go index ab29fc9c0..437cab433 100644 --- a/attr.go +++ b/attr.go @@ -16,13 +16,15 @@ package pilosa import ( "bytes" - "crypto/sha1" + "encoding/binary" "fmt" "sort" "sync" "time" + "github.com/cespare/xxhash" + "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -242,7 +244,7 @@ func (s *AttrStore) Blocks() ([]AttrBlock, error) { block := AttrBlock{ID: cur.blockID()} // Compute checksum of every key/value in block. - h := sha1.New() + h := xxhash.New() for k, v := cur.next(); k != nil; k, v = cur.next() { h.Write(k) h.Write(v) diff --git a/bitmap_test.go b/bitmap_test.go index 4fcd1b115..7eed299fc 100644 --- a/bitmap_test.go +++ b/bitmap_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "fmt" "reflect" "testing" @@ -23,14 +24,34 @@ import ( // Ensure a bitmap can be merged func TestBitmap_Merge(t *testing.T) { - bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth) - bm2 := pilosa.NewBitmap(3, 4, 5) - bm1.Merge(bm2) - - if bm1.Count() != 7 { - t.Fatalf("Count after merge %d != 7\n", bm1.Count()) + tests := []struct { + bm1 *pilosa.Bitmap + bm2 *pilosa.Bitmap + exp uint64 + }{ + { + bm1: pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth), + bm2: pilosa.NewBitmap(3, 4, 5), + exp: 7, + }, + { + bm1: pilosa.NewBitmap(), + bm2: pilosa.NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + exp: 7, + }, } + for i, test := range tests { + t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) { + test.bm1.Merge(test.bm2) + if cnt := test.bm1.Count(); cnt != test.exp { + t.Fatalf("merged count %d is not %d", cnt, test.exp) + } + if length := len(test.bm1.Bits()); uint64(length) != test.exp { + t.Fatalf("merged length %d is not %d", length, test.exp) + } + }) + } } // Ensure a bitmap can Xor'ed diff --git a/config_test.go b/config_test.go index 38a3fc640..1359e2e31 100644 --- a/config_test.go +++ b/config_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/ctl/common.go b/ctl/common.go index 826462b8c..9bba68657 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index eb761e6c9..60248d6bb 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package diagnostics import ( diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 7ac4906d4..8f85a57db 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package diagnostics_test import ( diff --git a/docs/administration.md b/docs/administration.md index 795d6f1c4..b77d05262 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - Host to receive events - - Polling interval for runtime metrics - - Metric type (StatsD, Expvar). + - [Host](../configuration#metrics-host): specify host that receives metric events + - [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration#metrics-service): declare type StatsD or Expvar ##### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: diff --git a/docs/api-reference.md b/docs/api-reference.md index 206c30f55..52ea2c369 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -13,13 +13,10 @@ nav = [] Returns the schema of all indexes in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index ``` - -Response: -``` +``` response {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` @@ -29,13 +26,10 @@ Response: Returns the schema of the specified index in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user ``` - -Response: -``` +``` response {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` @@ -45,13 +39,10 @@ Response: Creates an index with the given name. -Request: -``` +``` request curl -XPOST localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -61,13 +52,10 @@ Response: Removes the given index. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -77,15 +65,12 @@ Response: Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. -Request: -``` +``` request curl localhost:10101/index/user/query \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` - -Response: -``` +``` response {"results":[{"attrs":{},"bits":[100]}]} ``` @@ -95,14 +80,12 @@ The response doesn't include column attributes by default. To return them, set t The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -Request: -``` +``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` -Response: -``` +``` response { "results":[{"attrs":{},"bits":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] @@ -134,21 +117,21 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. -Request: -``` +``` request curl localhost:10101/index/user/frame/language \ -X POST \ -d '{"options": {"inverseEnabled": true}}' ``` - +``` response +{} ``` + +``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` - -Response: -``` +``` response {} ``` @@ -158,13 +141,10 @@ Response: Removes the given frame. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/frame/language ``` - -Response: -``` +``` response {} ``` @@ -188,15 +168,12 @@ The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Va * MDH: month, day and hour * YMDH: year, month, day and hour -Request: -``` +``` request curl localhost:10101/index/user/frame/language/time-quantum \ -X POST \ -d '{"timeQuantum": "YM"}' ``` - -Response: -``` +``` response {} ``` @@ -211,15 +188,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max` * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. -Request: -``` +``` request curl localhost:10101/index/repository/frame/stats/field/pullrequests \ -X POST \ -d '{"type": "int", "min": 0, "max": 1000000}' ``` - -Response: -``` +``` response {} ``` @@ -251,8 +225,7 @@ The `action` describes how the field value will be processed. Each `action` may - `mapping`: Map the value to a RowID in the `valueMap`. * `valueMap` (object): string and integer pairs used to map field values to RowID's. -Request: -``` +``` request curl localhost:10101/index/user/input-definition/stargazer-input \ -X POST \ -d '{ @@ -284,9 +257,7 @@ curl localhost:10101/index/user/input-definition/stargazer-input \ ] }' ``` - -Response: -``` +``` response {} ``` @@ -296,13 +267,10 @@ Response: Returns the given input definition as JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} ``` @@ -312,13 +280,10 @@ Response: Removes the given input definition. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {} ``` @@ -330,15 +295,12 @@ Processes the JSON payload using the given input definition. The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. -Request: -``` +``` request curl localhost:10101/index/user/input/stargazer-input \ -X POST \ -d '[{"language_id": "Go", "repo_id": 92274475}]' ``` - -Response: -``` +``` response {} ``` @@ -348,13 +310,10 @@ Response: Returns the hosts in the cluster. -Request: -``` +``` request curl -XGET localhost:10101/hosts ``` - -Response: -``` +``` response [{"host":":10101"}] ``` @@ -364,13 +323,10 @@ Response: Returns the version of the Pilosa server. -Request: -``` +``` request curl -XGET localhost:10101/version ``` - -Response: -``` +``` response {"version":"v0.6.0"} ``` diff --git a/docs/configuration.md b/docs/configuration.md index d9dc4739b..b6d7a603e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -219,7 +219,7 @@ Any flag that has a value that is a comma separated list on the command line bec cpu-time = "30s" ``` ##### Metric Service -* Description: Which stats service to use (StatsD or ExpVar). +* Description: Which stats service to use. Choose from [statsd, expvar]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: diff --git a/docs/getting-started.md b/docs/getting-started.md index 35042d59d..d4802e8ee 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,11 +31,12 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest ``` Let's make sure Pilosa is running: -``` +``` request curl localhost:10101/status ``` - -Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` +``` response +{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}} +``` ### Sample Project @@ -45,30 +46,41 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term #### Create the Schema -Note: The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: -``` +``` request curl localhost:10101/schema +``` +``` response {"indexes":null} ``` Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: -``` +``` request curl localhost:10101/index/repository -X POST ``` +``` response +{} +``` Let's create the `stargazer` frame which has user IDs of stargazers as its rows: -``` +``` request curl localhost:10101/index/repository/frame/stargazer \ -X POST \ -d '{"options": {"timeQuantum": "YMD"}}' ``` +``` response +{} +``` Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` frame, which will contain IDs for programming languages: +``` request +curl localhost:10101/index/repository/frame/language \ + -X POST ``` -curl localhost:10101/index/repository/frame/language -X POST +``` response +{} ``` #### Import Data From CSV Files @@ -107,46 +119,112 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d Which repositories did user 14 star: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'Bitmap(frame="stargazer", rowID=14)' ``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + } + ] +} +``` What are the top 5 languages in the sample data: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'TopN(frame="language", n=5)' ``` +``` response +{ + "results":[ + [ + {"id":5,"count":119}, + {"id":1,"count":50}, + {"id":4,"count":48}, + {"id":9,"count":31}, + {"id":13,"count":25} + ] + ] +} +``` Which repositories were starred by user 14 and 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,3,362,396,416,461,464,466,470,486] + } + ] +} ``` Which repositories were starred by user 14 or 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Union( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + } + ] +} ``` Which repositories were starred by user 14 and 19 and also were written in language 1: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19), + Bitmap(frame="language", rowID=1) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,362,416,461] + } + ] +} ``` Set user 99999 as a stargazer for repository 77777: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' ``` +``` response +{"results":[true]} +``` ### What's Next? diff --git a/docs/query-language.md b/docs/query-language.md index f28257fe6..0e02351ab 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -34,11 +34,14 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' ``` +``` response +{"results":[true]} +``` #### Arguments and Types diff --git a/docs/tutorials.md b/docs/tutorials.md index 9c19f3a67..4d787577c 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -178,38 +178,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. All nodes should be in the `UP` state: -``` +``` response {"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} ``` #### Running Queries Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` +``` response +{} +``` This will create index `sample-index` with default options. Let's create the frame now: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' ``` +``` response +{} +``` We just created frame `sample-frame` with default options. Let's run a `SetBit` query: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' ``` +``` response +{"results":[true]} +``` Confirm that the bit was indeed set: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` The same response should be returned when querying other nodes in the cluster: -``` +``` request curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` #### What's Next? @@ -223,20 +238,16 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: -``` +``` request curl localhost:10101/index/patients \ -X POST ``` - -Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient. -``` -curl localhost:10101/index/patients/frame/measurements \ - -X POST \ - -d '{"options":{"rangeEnabled": true}}' +``` response +{} ``` In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. -``` +``` request curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ @@ -248,17 +259,23 @@ curl localhost:10101/index/patients/frame/measurements \ ] }}' ``` +``` response +{} +``` If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' ``` +``` response +{"results":[null]} +``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -282,39 +299,38 @@ pilosa import -i patients -f measurements --field age ages.csv Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Range(frame="measurements", age > 40)' ``` -You should get the following results: -``` +``` response {"results":[{"attrs":{},"bits":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(frame="measurements", field="age")' ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -``` +``` response {"results":[{"sum":377,"count":9}]} ``` +The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' ``` -Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. -``` +``` response {"results":[{"sum":191,"count":3}]} ``` +Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. ### Storing Row and Column Attributes @@ -323,20 +339,26 @@ Notice in this case that the count is only `3` because of the `age > 40` filter Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data. First, create an index called `books` to use for this tutorial: -``` +``` request curl localhost:10101/index/books \ -X POST ``` +``` response +{} +``` Next, create a frame in the `books` index called `members` which will represent library members who have read books. -``` +``` request curl localhost:10101/index/books/frame/members \ -X POST \ -d '{}' ``` +``` response +{} +``` Now, let's add some books to our index. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) @@ -345,9 +367,12 @@ curl localhost:10101/index/books/query \ SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' ``` +``` response +{"results":[null,null,null,null,null]} +``` And add some members. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") @@ -356,20 +381,22 @@ curl localhost:10101/index/books/query \ SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' ``` +``` response +{"results":[null,null,null,null,null]} +``` At this point we can query one of the `member` records by querying that row. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -You should get the following result set: -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetBit(frame="members", rowID=10001, columnID=3) @@ -390,28 +417,28 @@ curl localhost:10101/index/books/query \ SetBit(frame="members", rowID=10005, columnID=4) SetBit(frame="members", rowID=10005, columnID=5)' ``` +``` response +{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} +``` Now pull the record for `Sue Perkins` again. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} ``` +Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` - -Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute. - -``` +``` response { "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], "columnAttrs":[ @@ -421,15 +448,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt ] } ``` +The `book` attributes are included in the result set at the `columnAttrs` attribute. Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members: -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' ``` - -``` +``` response { "results":[{"attrs":{},"bits":[4]}], "columnAttrs":[ diff --git a/fragment.go b/fragment.go index 3b09b925e..40a42dbe5 100644 --- a/fragment.go +++ b/fragment.go @@ -20,7 +20,6 @@ import ( "bytes" "container/heap" "context" - "crypto/sha1" "encoding/binary" "errors" "fmt" @@ -36,6 +35,8 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash" + "math" "github.com/gogo/protobuf/proto" @@ -1021,7 +1022,7 @@ type TopOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { - h := sha1.New() + h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) } @@ -1665,7 +1666,7 @@ type blockHasher struct { func newBlockHasher() blockHasher { return blockHasher{ blockID: -1, - hash: sha1.New(), + hash: xxhash.New(), } } func (h *blockHasher) Reset() { diff --git a/fragment_test.go b/fragment_test.go index 2ccf538f7..8a54d3b07 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1075,3 +1075,26 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } + +func BenchmarkFragment_Snapshot(b *testing.B) { + if *FragmentPath == "" { + b.Skip("no fragment specified") + } + + // Open the fragment specified by the path. + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + if err := f.Open(); err != nil { + b.Fatal(err) + } + defer f.Close() + + // Reset timer and execute benchmark. + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + err := f.Snapshot() + if err != nil { + b.Fatalf("unexpected count (reopen): %s", err) + } + } +} diff --git a/handler_test.go b/handler_test.go index 9c8ce2bf4..c8030a9ea 100644 --- a/handler_test.go +++ b/handler_test.go @@ -377,7 +377,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } @@ -465,7 +465,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { @@ -526,7 +526,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -599,7 +599,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } diff --git a/pilosa_test.go b/pilosa_test.go index b4d9cc2e2..d82e27fa3 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/roaring/roaring.go b/roaring/roaring.go index 055c81e36..8d88b2088 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -840,6 +840,10 @@ func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) } // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { + // k should always be -1 unless we're seeking into a run container. Then the + // "if c.isRun" section will take care of it. + itr.k = -1 + // Move to the correct container. itr.i = search64(itr.bitmap.keys, highbits(seek)) if itr.i < 0 { @@ -2177,7 +2181,7 @@ func unionArrayArray(a, b *container) *container { // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } output := &container{containerType: ContainerRun} @@ -2234,10 +2238,10 @@ func (c *container) runAppendInterval(v interval16) int { } func unionRunRun(a, b *container) *container { - if a.n == maxContainerVal { + if a.n == maxContainerVal+1 { return a.clone() } - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } na, nb := len(a.runs), len(b.runs) @@ -2268,9 +2272,12 @@ func unionRunRun(a, b *container) *container { } func unionBitmapRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } + if a.n == maxContainerVal+1 { + return a.clone() + } output := a.clone() for j := 0; j < len(b.runs); j++ { output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) @@ -2285,7 +2292,7 @@ func (c *container) bitmapSetRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcnt(X) ycnt := popcnt(Y) if x == y { @@ -2308,7 +2315,7 @@ func (c *container) bitmapXorRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { cnt := popcnt(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip @@ -2333,7 +2340,7 @@ func (c *container) bitmapZeroRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { c.n -= int(popcnt(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) @@ -2469,10 +2476,18 @@ func differenceArrayRun(a, b *container) *container { if i < len(a.array) { // keep all array elements after end of runs - output.array = append(output.array, a.array[i:]...) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += len(a.array[i:]) + // It's possible that output was converted from array to bitmap in output.add() + // so check container type before proceeding. + if output.containerType == ContainerArray { + output.array = append(output.array, a.array[i:]...) + // TODO: consider handling container.n mutations in one place + // like we do with container.add(). + output.n += len(a.array[i:]) + } else { + for _, v := range a.array[i:] { + output.add(v) + } + } } return output } @@ -2500,6 +2515,8 @@ func differenceRunArray(a, b *container) *container { bidx := 0 vb := b.array[bidx] + +RUNLOOP: for _, run := range a.runs { start := run.start for vb < run.start { @@ -2511,6 +2528,9 @@ func differenceRunArray(a, b *container) *container { } for vb >= run.start && vb <= run.last { if vb == start { + if vb == 65535 { // overflow + break RUNLOOP + } start++ bidx++ if bidx >= len(b.array) { @@ -2521,6 +2541,9 @@ func differenceRunArray(a, b *container) *container { } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) output.n += int(vb - start) + if vb == 65535 { // overflow + break RUNLOOP + } start = vb + 1 bidx++ if bidx >= len(b.array) { @@ -2772,7 +2795,9 @@ func xorArrayBitmap(a, b *container) *container { } } - if output.count() < ArrayMaxSize { + // It's possible that output was converted from bitmap to array in output.remove() + // so we only do this conversion if output is still a bitmap container. + if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3187,7 +3212,7 @@ func xorRunRun(a, b *container) *container { if nb == 0 { return a.clone() } - output := &container{} + output := &container{containerType: ContainerRun} lastI, lastJ := -1, -1 diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go new file mode 100644 index 000000000..ea6d86cd9 --- /dev/null +++ b/roaring/roaring_helpers_test.go @@ -0,0 +1,295 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +/////////////////////////////////////////////////////////////////////////// + +var containerWidth uint64 = 65536 + +////////////////// array +func arrayEmpty() []uint16 { + return make([]uint16, 0) +} + +func arrayFull() []uint16 { + array := make([]uint16, containerWidth) + for i := 0; i < int(containerWidth); i++ { + array[i] = uint16(i) + } + return array +} + +func arrayFirstBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + return array +} + +func arrayLastBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(65535)) + return array +} + +func arrayFirstBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 1; i < int(containerWidth); i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayLastBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 0; i < int(containerWidth)-1; i++ { + array[i] = uint16(i) + } + return array +} + +func arrayInnerBitsSet() []uint16 { + array := make([]uint16, containerWidth-2) + for i := 1; i < int(containerWidth)-1; i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayOuterBitsSet() []uint16 { + return []uint16{0, 65535} +} + +func arrayOddBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2*i + 1) + } + return array +} + +func arrayEvenBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2 * i) + } + return array +} + +////////////////// bitmap +func bitmapEmpty() []uint64 { + return make([]uint64, bitmapN) +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapFirstBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + return bitmap +} + +func bitmapLastBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitUnset() []uint64 { + bitmap := bitmapFull() + bitmap[0] = 0xFFFFFFFFFFFFFFFE + return bitmap +} + +func bitmapLastBitUnset() []uint64 { + bitmap := bitmapFull() + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapInnerBitsSet() []uint64 { + bitmap := bitmapFull() + bitmap[0] = 0xFFFFFFFFFFFFFFFE + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapOuterBitsSet() []uint64 { + bitmap := bitmapEmpty() + bitmap[0] = 0x0000000000000001 + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapOddBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xAAAAAAAAAAAAAAAA + } + return bitmap +} + +func bitmapEvenBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0x5555555555555555 + } + return bitmap +} + +////////////////// run +func runEmpty() []interval16 { + return make([]interval16, 0) +} + +func runFull() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65535}) + return run +} + +func runFirstBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + return run +} + +func runLastBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runFirstBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65535}) + return run +} + +func runLastBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65534}) + return run +} + +func runInnerBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65534}) + return run +} + +func runOuterBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runOddBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)} + } + return run +} + +func runEvenBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)} + } + return run +} + +/////////////////////////////////////////////////////////////////////////// + +type testOp struct { + f func(a, b *container) *container + x string + y string + exp string +} + +func doContainer(containerType byte, data interface{}) *container { + c := &container{ + containerType: containerType, + } + + switch containerType { + case ContainerArray: + c.array = data.([]uint16) + case ContainerBitmap: + c.bitmap = data.([]uint64) + case ContainerRun: + c.runs = data.([]interval16) + } + c.n = c.count() + + return c +} + +func setupContainerTests() map[byte]map[string]*container { + + cts := make(map[byte]map[string]*container) + + // array containers + cts[ContainerArray] = map[string]*container{ + "empty": doContainer(ContainerArray, arrayEmpty()), + "full": doContainer(ContainerArray, arrayFull()), + "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), + } + + // bitmap containers + cts[ContainerBitmap] = map[string]*container{ + "empty": doContainer(ContainerBitmap, bitmapEmpty()), + "full": doContainer(ContainerBitmap, bitmapFull()), + "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), + } + + // run containers + cts[ContainerRun] = map[string]*container{ + "empty": doContainer(ContainerRun, runEmpty()), + "full": doContainer(ContainerRun, runFull()), + "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), + "lastBitSet": doContainer(ContainerRun, runLastBitSet()), + "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), + "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), + "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), + } + + return cts +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 4b2813908..dca413811 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -18,6 +18,8 @@ import ( "bytes" "fmt" "reflect" + "runtime" + "strings" "testing" ) @@ -1479,6 +1481,16 @@ func TestDifferenceRunArray(t *testing.T) { array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27}, exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}}, }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}}, + array: []uint16{65533, 65534, 65535}, + exp: []interval16{{start: 0, last: 20}}, + }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}}, + array: []uint16{37, 65535}, + exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}}, + }, } for i, test := range tests { a.runs = test.runs @@ -1583,14 +1595,69 @@ func TestDifferenceBitmapRun(t *testing.T) { runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, exp: []uint64{0xFFFF0000FFFFFF0F}, }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}}, + exp: []uint64{0xFFFFFFFFFFFFFF80}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 63}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8}, + }, + { + bitmap: []uint64{0x0000000000000001}, + runs: []interval16{{start: 0, last: 0}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0x8000000000000000}, + runs: []interval16{{start: 63, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xC000000000000000, 0x0000000000000003}, + runs: []interval16{{start: 63, last: 64}}, + exp: []uint64{0x4000000000000000, 0x0000000000000002}, + }, + { + bitmap: []uint64{0x0000000000000000}, + runs: []interval16{{start: 5, last: 7}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: bitmapLastBitSet(), + runs: []interval16{{start: 65535, last: 65535}}, + exp: bitmapEmpty(), + }, + { + bitmap: bitmapFull(), + runs: []interval16{{start: 0, last: 65535}}, + exp: bitmapEmpty(), + }, } for i, test := range tests { for i, v := range test.bitmap { a.bitmap[i] = v } - a.n = a.bitmapCountRange(0, 100) + a.n = a.bitmapCountRange(0, 65536) b.runs = test.runs - b.n = b.runCountRange(0, 100) + b.n = b.runCountRange(0, 65536) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) @@ -1622,12 +1689,12 @@ func TestDifferenceBitmapArray(t *testing.T) { exp: []uint16{8, 9, 11, 12, 13, 14, 15}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10}, exp: []uint16{9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{63}, exp: []uint16{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61}, }, @@ -2244,6 +2311,44 @@ func TestIteratorRuns(t *testing.T) { } } +func TestIteratorVarious(t *testing.T) { + tests := []struct { + bm *Bitmap + exp uint64 + }{ + { + bm: NewBitmap(3, 4, 5), + exp: 3, + }, + { + bm: bitmapVariousContainers(), + exp: 61221, + }, + { + bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + exp: 7, + }, + } + + for i, test := range tests { + test.bm.Optimize() + t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) { + if cnt := test.bm.Count(); cnt != test.exp { + t.Fatalf("merged count %d is not %d", cnt, test.exp) + } + iter := test.bm.Iterator() + bits := make([]uint64, 0, test.bm.Count()) + for v, eof := iter.Next(); !eof; v, eof = iter.Next() { + bits = append(bits, v) + } + if length := len(bits); uint64(length) != test.exp { + t.Fatalf("length %d is not %d", length, test.exp) + } + }) + } + +} + func TestRunBinSearchContains(t *testing.T) { tests := []struct { runs []interval16 @@ -2390,7 +2495,7 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { } } -func TestSearc64(t *testing.T) { +func TestSearch64(t *testing.T) { tests := []struct { a []uint64 value uint64 @@ -2505,12 +2610,12 @@ func TestIntersectArrayBitmap(t *testing.T) { }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), exp: []uint16{1, 63, 543, 639, 65535}, }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapEvens(), + bitmap: bitmapEvenBitsSet(), exp: []uint16{0, 120, 12000, 65534}, }, } @@ -2532,18 +2637,606 @@ func TestIntersectArrayBitmap(t *testing.T) { } } -func bitmapOdds() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xAAAAAAAAAAAAAAAA +// rleCont returns a slice of numbers all in the range starting from +// container_width*num, and ending at container_width*(num+1)-1. If left is +// true, then the first 100 bits will be set, if mid is true, 100 bits in the +// middle will be set, if right is true, the last 100 bits will be set. +// sets 100 bits per true +func rleCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + for i := uint64(0); i < 100; i++ { + ret = append(ret, base+i) + } } - return bitmap + if mid { + for i := containerWidth / 2; i < containerWidth/2+100; i++ { + ret = append(ret, base+i) + } + } + if right { + for i := containerWidth - 100; i < containerWidth; i++ { + ret = append(ret, base+i) + } + } + return ret } -func bitmapEvens() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0x5555555555555555 +// sets 2 bits per true. +func arrCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + ret = append(ret, base+0, base+2) + } + if mid { + half := containerWidth / 2 + ret = append(ret, base+half, base+half+2) + } + if right { + ret = append(ret, base+containerWidth-3, base+containerWidth-1) + } + return ret +} + +// sets 6667 bits per true. +func bitCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + for i := uint64(0); i < 20001; i += 3 { + ret = append(ret, base+i) + } + } + if mid { + for i := uint64(21000); i < 41001; i += 3 { + ret = append(ret, base+i) + } + } + if right { + for i := uint64(45537); i <= 65535; i += 3 { + ret = append(ret, base+i) + } + } + return ret +} + +func bitmapVariousContainers() *Bitmap { + bits := make([]uint64, 0) + bits = append(bits, rleCont(0, true, true, true)...) + bits = append(bits, rleCont(1, true, true, true)...) + bits = append(bits, arrCont(2, true, true, true)...) + bits = append(bits, arrCont(3, true, true, true)...) + bits = append(bits, bitCont(4, true, true, true)...) + bits = append(bits, bitCont(5, true, true, true)...) + bits = append(bits, rleCont(6, true, true, true)...) + bits = append(bits, bitCont(7, true, true, true)...) + bits = append(bits, arrCont(8, true, true, true)...) + bits = append(bits, rleCont(9, true, true, true)...) + bm := NewBitmap(bits...) + bm.Optimize() + return bm +} + +/////////////////////////////////////////////////////////////////////////// + +func getFunctionName(i interface{}) string { + x := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + y := strings.Split(x, ".") + y = y[len(y)-1:] + return y[0] +} + +func TestContainerCombinations(t *testing.T) { + + cts := setupContainerTests() + + containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + + // map used for a more descriptive print + cm := map[byte]string{ + ContainerArray: "array", + ContainerBitmap: "bitmap", + ContainerRun: "run", + } + + testOps := []testOp{ + // intersect + {intersect, "empty", "empty", "empty"}, + {intersect, "empty", "full", "empty"}, + {intersect, "empty", "firstBitSet", "empty"}, + {intersect, "empty", "lastBitSet", "empty"}, + {intersect, "empty", "firstBitUnset", "empty"}, + {intersect, "empty", "lastBitUnset", "empty"}, + {intersect, "empty", "innerBitsSet", "empty"}, + {intersect, "empty", "outerBitsSet", "empty"}, + {intersect, "empty", "oddBitsSet", "empty"}, + {intersect, "empty", "evenBitsSet", "empty"}, + // + {intersect, "full", "empty", "empty"}, + {intersect, "full", "full", "full"}, + {intersect, "full", "firstBitSet", "firstBitSet"}, + {intersect, "full", "lastBitSet", "lastBitSet"}, + {intersect, "full", "firstBitUnset", "firstBitUnset"}, + {intersect, "full", "lastBitUnset", "lastBitUnset"}, + {intersect, "full", "innerBitsSet", "innerBitsSet"}, + {intersect, "full", "outerBitsSet", "outerBitsSet"}, + {intersect, "full", "oddBitsSet", "oddBitsSet"}, + {intersect, "full", "evenBitsSet", "evenBitsSet"}, + // + {intersect, "firstBitSet", "empty", "empty"}, + {intersect, "firstBitSet", "full", "firstBitSet"}, + {intersect, "firstBitSet", "firstBitSet", "firstBitSet"}, + {intersect, "firstBitSet", "lastBitSet", "empty"}, + {intersect, "firstBitSet", "firstBitUnset", "empty"}, + {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, + {intersect, "firstBitSet", "innerBitsSet", "empty"}, + {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + {intersect, "firstBitSet", "oddBitsSet", "empty"}, + {intersect, "firstBitSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "lastBitSet", "empty", "empty"}, + {intersect, "lastBitSet", "full", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitSet", "empty"}, + {intersect, "lastBitSet", "lastBitSet", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitUnset", "lastBitSet"}, + {intersect, "lastBitSet", "lastBitUnset", "empty"}, + {intersect, "lastBitSet", "innerBitsSet", "empty"}, + {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "oddBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "evenBitsSet", "empty"}, + // + {intersect, "firstBitUnset", "empty", "empty"}, + {intersect, "firstBitUnset", "full", "firstBitUnset"}, + {intersect, "firstBitUnset", "firstBitSet", "empty"}, + {intersect, "firstBitUnset", "lastBitSet", "lastBitSet"}, + {intersect, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, + {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + {intersect, "firstBitUnset", "oddBitsSet", "oddBitsSet"}, + //{intersect, "firstBitUnset", "evenBitsSet", ""}, + // + {intersect, "lastBitUnset", "empty", "empty"}, + {intersect, "lastBitUnset", "full", "lastBitUnset"}, + {intersect, "lastBitUnset", "firstBitSet", "firstBitSet"}, + {intersect, "lastBitUnset", "lastBitSet", "empty"}, + {intersect, "lastBitUnset", "firstBitUnset", "innerBitsSet"}, + {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + //{intersect, "lastBitUnset", "oddBitsSet", ""}, + {intersect, "lastBitUnset", "evenBitsSet", "evenBitsSet"}, + // + {intersect, "innerBitsSet", "empty", "empty"}, + {intersect, "innerBitsSet", "full", "innerBitsSet"}, + {intersect, "innerBitsSet", "firstBitSet", "empty"}, + {intersect, "innerBitsSet", "lastBitSet", "empty"}, + {intersect, "innerBitsSet", "firstBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + //{intersect, "innerBitsSet", "oddBitsSet", ""}, + //{intersect, "innerBitsSet", "evenBitsSet", ""}, + // + {intersect, "outerBitsSet", "empty", "empty"}, + {intersect, "outerBitsSet", "full", "outerBitsSet"}, + {intersect, "outerBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "outerBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "outerBitsSet", "firstBitUnset", "lastBitSet"}, + {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, + {intersect, "outerBitsSet", "innerBitsSet", "empty"}, + {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + {intersect, "outerBitsSet", "oddBitsSet", "lastBitSet"}, + {intersect, "outerBitsSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "oddBitsSet", "empty", "empty"}, + {intersect, "oddBitsSet", "full", "oddBitsSet"}, + {intersect, "oddBitsSet", "firstBitSet", "empty"}, + {intersect, "oddBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "oddBitsSet", "firstBitUnset", "oddBitsSet"}, + //{intersect, "oddBitsSet", "lastBitUnset", ""}, + //{intersect, "oddBitsSet", "innerBitsSet", ""}, + {intersect, "oddBitsSet", "outerBitsSet", "lastBitSet"}, + {intersect, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {intersect, "oddBitsSet", "evenBitsSet", "empty"}, + // + {intersect, "evenBitsSet", "empty", "empty"}, + {intersect, "evenBitsSet", "full", "evenBitsSet"}, + {intersect, "evenBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "evenBitsSet", "lastBitSet", "empty"}, + //{intersect, "evenBitsSet", "firstBitUnset", ""}, + {intersect, "evenBitsSet", "lastBitUnset", "evenBitsSet"}, + //{intersect, "evenBitsSet", "innerBitsSet", ""}, + {intersect, "evenBitsSet", "outerBitsSet", "firstBitSet"}, + {intersect, "evenBitsSet", "oddBitsSet", "empty"}, + {intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, + + // union + {union, "empty", "empty", "empty"}, + {union, "empty", "full", "full"}, + {union, "empty", "firstBitSet", "firstBitSet"}, + {union, "empty", "lastBitSet", "lastBitSet"}, + {union, "empty", "firstBitUnset", "firstBitUnset"}, + {union, "empty", "lastBitUnset", "lastBitUnset"}, + {union, "empty", "innerBitsSet", "innerBitsSet"}, + {union, "empty", "outerBitsSet", "outerBitsSet"}, + {union, "empty", "oddBitsSet", "oddBitsSet"}, + {union, "empty", "evenBitsSet", "evenBitsSet"}, + // + {union, "full", "empty", "full"}, + {union, "full", "full", "full"}, + {union, "full", "firstBitSet", "full"}, + {union, "full", "lastBitSet", "full"}, + {union, "full", "firstBitUnset", "full"}, + {union, "full", "lastBitUnset", "full"}, + {union, "full", "innerBitsSet", "full"}, + {union, "full", "outerBitsSet", "full"}, + {union, "full", "oddBitsSet", "full"}, + {union, "full", "evenBitsSet", "full"}, + // + {union, "firstBitSet", "empty", "firstBitSet"}, + {union, "firstBitSet", "full", "full"}, + {union, "firstBitSet", "firstBitSet", "firstBitSet"}, + {union, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {union, "firstBitSet", "firstBitUnset", "full"}, + {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, + {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + //{union, "firstBitSet", "oddBitsSet", ""}, + {union, "firstBitSet", "evenBitsSet", "evenBitsSet"}, + // + {union, "lastBitSet", "empty", "lastBitSet"}, + {union, "lastBitSet", "full", "full"}, + {union, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {union, "lastBitSet", "lastBitSet", "lastBitSet"}, + {union, "lastBitSet", "firstBitUnset", "firstBitUnset"}, + {union, "lastBitSet", "lastBitUnset", "full"}, + {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + {union, "lastBitSet", "oddBitsSet", "oddBitsSet"}, + //{union, "lastBitSet", "evenBitsSet", ""}, + // + {union, "firstBitUnset", "empty", "firstBitUnset"}, + {union, "firstBitUnset", "full", "full"}, + {union, "firstBitUnset", "firstBitSet", "full"}, + {union, "firstBitUnset", "lastBitSet", "firstBitUnset"}, + {union, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {union, "firstBitUnset", "lastBitUnset", "full"}, + {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "outerBitsSet", "full"}, + {union, "firstBitUnset", "oddBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "evenBitsSet", "full"}, + // + {union, "lastBitUnset", "empty", "lastBitUnset"}, + {union, "lastBitUnset", "full", "full"}, + {union, "lastBitUnset", "firstBitSet", "lastBitUnset"}, + {union, "lastBitUnset", "lastBitSet", "full"}, + {union, "lastBitUnset", "firstBitUnset", "full"}, + {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, + {union, "lastBitUnset", "outerBitsSet", "full"}, + {union, "lastBitUnset", "oddBitsSet", "full"}, + {union, "lastBitUnset", "evenBitsSet", "lastBitUnset"}, + // + {union, "innerBitsSet", "empty", "innerBitsSet"}, + {union, "innerBitsSet", "full", "full"}, + {union, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {union, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {union, "innerBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {union, "innerBitsSet", "outerBitsSet", "full"}, + {union, "innerBitsSet", "oddBitsSet", "firstBitUnset"}, + {union, "innerBitsSet", "evenBitsSet", "lastBitUnset"}, + // + {union, "outerBitsSet", "empty", "outerBitsSet"}, + {union, "outerBitsSet", "full", "full"}, + {union, "outerBitsSet", "firstBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "lastBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "firstBitUnset", "full"}, + {union, "outerBitsSet", "lastBitUnset", "full"}, + {union, "outerBitsSet", "innerBitsSet", "full"}, + {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + //{union, "outerBitsSet", "oddBitsSet", ""}, + //{union, "outerBitsSet", "evenBitsSet", ""}, + // + {union, "oddBitsSet", "empty", "oddBitsSet"}, + {union, "oddBitsSet", "full", "full"}, + //{union, "oddBitsSet", "firstBitSet", ""}, + {union, "oddBitsSet", "lastBitSet", "oddBitsSet"}, + {union, "oddBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "oddBitsSet", "lastBitUnset", "full"}, + {union, "oddBitsSet", "innerBitsSet", "firstBitUnset"}, + //{union, "oddBitsSet", "outerBitsSet", ""}, + {union, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {union, "oddBitsSet", "evenBitsSet", "full"}, + // + {union, "evenBitsSet", "empty", "evenBitsSet"}, + {union, "evenBitsSet", "full", "full"}, + {union, "evenBitsSet", "firstBitSet", "evenBitsSet"}, + //{union, "evenBitsSet", "lastBitSet", ""}, + {union, "evenBitsSet", "firstBitUnset", "full"}, + {union, "evenBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "evenBitsSet", "innerBitsSet", "lastBitUnset"}, + //{union, "evenBitsSet", "outerBitsSet", ""}, + {union, "evenBitsSet", "oddBitsSet", "full"}, + {union, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, + + // difference + {difference, "empty", "empty", "empty"}, + {difference, "empty", "full", "empty"}, + {difference, "empty", "firstBitSet", "empty"}, + {difference, "empty", "lastBitSet", "empty"}, + {difference, "empty", "firstBitUnset", "empty"}, + {difference, "empty", "lastBitUnset", "empty"}, + {difference, "empty", "innerBitsSet", "empty"}, + {difference, "empty", "outerBitsSet", "empty"}, + {difference, "empty", "oddBitsSet", "empty"}, + {difference, "empty", "evenBitsSet", "empty"}, + // + {difference, "full", "empty", "full"}, + {difference, "full", "full", "empty"}, + {difference, "full", "firstBitSet", "firstBitUnset"}, + {difference, "full", "lastBitSet", "lastBitUnset"}, + {difference, "full", "firstBitUnset", "firstBitSet"}, + {difference, "full", "lastBitUnset", "lastBitSet"}, + {difference, "full", "innerBitsSet", "outerBitsSet"}, + {difference, "full", "outerBitsSet", "innerBitsSet"}, + {difference, "full", "oddBitsSet", "evenBitsSet"}, + {difference, "full", "evenBitsSet", "oddBitsSet"}, + // + {difference, "firstBitSet", "empty", "firstBitSet"}, + {difference, "firstBitSet", "full", "empty"}, + {difference, "firstBitSet", "firstBitSet", "empty"}, + {difference, "firstBitSet", "lastBitSet", "firstBitSet"}, + {difference, "firstBitSet", "firstBitUnset", "firstBitSet"}, + {difference, "firstBitSet", "lastBitUnset", "empty"}, + {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "outerBitsSet", "empty"}, + {difference, "firstBitSet", "oddBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "evenBitsSet", "empty"}, + // + {difference, "lastBitSet", "empty", "lastBitSet"}, + {difference, "lastBitSet", "full", "empty"}, + {difference, "lastBitSet", "firstBitSet", "lastBitSet"}, + {difference, "lastBitSet", "lastBitSet", "empty"}, + {difference, "lastBitSet", "firstBitUnset", "empty"}, + {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, + {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, + {difference, "lastBitSet", "outerBitsSet", "empty"}, + {difference, "lastBitSet", "oddBitsSet", "empty"}, + {difference, "lastBitSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "firstBitUnset", "empty", "firstBitUnset"}, + {difference, "firstBitUnset", "full", "empty"}, + {difference, "firstBitUnset", "firstBitSet", "firstBitUnset"}, + {difference, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {difference, "firstBitUnset", "firstBitUnset", "empty"}, + {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, + {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + //{difference, "firstBitUnset", "oddBitsSet", ""}, + {difference, "firstBitUnset", "evenBitsSet", "oddBitsSet"}, + // + {difference, "lastBitUnset", "empty", "lastBitUnset"}, + {difference, "lastBitUnset", "full", "empty"}, + {difference, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {difference, "lastBitUnset", "lastBitSet", "lastBitUnset"}, + {difference, "lastBitUnset", "firstBitUnset", "firstBitSet"}, + {difference, "lastBitUnset", "lastBitUnset", "empty"}, + {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + {difference, "lastBitUnset", "oddBitsSet", "evenBitsSet"}, + //{difference, "lastBitUnset", "evenBitsSet", ""}, + // + {difference, "innerBitsSet", "empty", "innerBitsSet"}, + {difference, "innerBitsSet", "full", "empty"}, + {difference, "innerBitsSet", "firstBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "lastBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "firstBitUnset", "empty"}, + {difference, "innerBitsSet", "lastBitUnset", "empty"}, + {difference, "innerBitsSet", "innerBitsSet", "empty"}, + {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + //{difference, "innerBitsSet", "oddBitsSet", ""}, + //{difference, "innerBitsSet", "evenBitsSet", ""}, + // + {difference, "outerBitsSet", "empty", "outerBitsSet"}, + {difference, "outerBitsSet", "full", "empty"}, + {difference, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {difference, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {difference, "outerBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, + {difference, "outerBitsSet", "outerBitsSet", "empty"}, + {difference, "outerBitsSet", "oddBitsSet", "firstBitSet"}, + {difference, "outerBitsSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "oddBitsSet", "empty", "oddBitsSet"}, + {difference, "oddBitsSet", "full", "empty"}, + {difference, "oddBitsSet", "firstBitSet", "oddBitsSet"}, + //{difference, "oddBitsSet", "lastBitSet", ""}, + {difference, "oddBitsSet", "firstBitUnset", "empty"}, + {difference, "oddBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "oddBitsSet", "innerBitsSet", "lastBitSet"}, + //{difference, "oddBitsSet", "outerBitsSet", ""}, + {difference, "oddBitsSet", "oddBitsSet", "empty"}, + {difference, "oddBitsSet", "evenBitsSet", "oddBitsSet"}, + // + {difference, "evenBitsSet", "empty", "evenBitsSet"}, + {difference, "evenBitsSet", "full", "empty"}, + //{difference, "evenBitsSet", "firstBitSet", ""}, + {difference, "evenBitsSet", "lastBitSet", "evenBitsSet"}, + {difference, "evenBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "evenBitsSet", "lastBitUnset", "empty"}, + {difference, "evenBitsSet", "innerBitsSet", "firstBitSet"}, + //{difference, "evenBitsSet", "outerBitsSet", ""}, + {difference, "evenBitsSet", "oddBitsSet", "evenBitsSet"}, + {difference, "evenBitsSet", "evenBitsSet", "empty"}, + + // xor + {xor, "empty", "empty", "empty"}, + {xor, "empty", "full", "full"}, + {xor, "empty", "firstBitSet", "firstBitSet"}, + {xor, "empty", "lastBitSet", "lastBitSet"}, + {xor, "empty", "firstBitUnset", "firstBitUnset"}, + {xor, "empty", "lastBitUnset", "lastBitUnset"}, + {xor, "empty", "innerBitsSet", "innerBitsSet"}, + {xor, "empty", "outerBitsSet", "outerBitsSet"}, + {xor, "empty", "oddBitsSet", "oddBitsSet"}, + {xor, "empty", "evenBitsSet", "evenBitsSet"}, + // + {xor, "full", "empty", "full"}, + {xor, "full", "full", "empty"}, + {xor, "full", "firstBitSet", "firstBitUnset"}, + {xor, "full", "lastBitSet", "lastBitUnset"}, + {xor, "full", "firstBitUnset", "firstBitSet"}, + {xor, "full", "lastBitUnset", "lastBitSet"}, + {xor, "full", "innerBitsSet", "outerBitsSet"}, + {xor, "full", "outerBitsSet", "innerBitsSet"}, + {xor, "full", "oddBitsSet", "evenBitsSet"}, + {xor, "full", "evenBitsSet", "oddBitsSet"}, + // + {xor, "firstBitSet", "empty", "firstBitSet"}, + {xor, "firstBitSet", "full", "firstBitUnset"}, + {xor, "firstBitSet", "firstBitSet", "empty"}, + {xor, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {xor, "firstBitSet", "firstBitUnset", "full"}, + {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, + {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + //{xor, "firstBitSet", "oddBitsSet", ""}, + //{xor, "firstBitSet", "evenBitsSet", ""}, + // + {xor, "lastBitSet", "empty", "lastBitSet"}, + {xor, "lastBitSet", "full", "lastBitUnset"}, + {xor, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {xor, "lastBitSet", "lastBitSet", "empty"}, + {xor, "lastBitSet", "firstBitUnset", "innerBitsSet"}, + {xor, "lastBitSet", "lastBitUnset", "full"}, + {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + //{xor, "lastBitSet", "oddBitsSet", ""}, + //{xor, "lastBitSet", "evenBitsSet", ""}, + // + {xor, "firstBitUnset", "empty", "firstBitUnset"}, + {xor, "firstBitUnset", "full", "firstBitSet"}, + {xor, "firstBitUnset", "firstBitSet", "full"}, + {xor, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {xor, "firstBitUnset", "firstBitUnset", "empty"}, + {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, + {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + //{xor, "firstBitUnset", "oddBitsSet", ""}, + //{xor, "firstBitUnset", "evenBitsSet", ""}, + // + {xor, "lastBitUnset", "empty", "lastBitUnset"}, + {xor, "lastBitUnset", "full", "lastBitSet"}, + {xor, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {xor, "lastBitUnset", "lastBitSet", "full"}, + {xor, "lastBitUnset", "firstBitUnset", "outerBitsSet"}, + {xor, "lastBitUnset", "lastBitUnset", "empty"}, + {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + //{xor, "lastBitUnset", "oddBitsSet", ""}, + //{xor, "lastBitUnset", "evenBitsSet", ""}, + // + {xor, "innerBitsSet", "empty", "innerBitsSet"}, + {xor, "innerBitsSet", "full", "outerBitsSet"}, + {xor, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {xor, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {xor, "innerBitsSet", "firstBitUnset", "lastBitSet"}, + {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, + {xor, "innerBitsSet", "innerBitsSet", "empty"}, + {xor, "innerBitsSet", "outerBitsSet", "full"}, + //{xor, "innerBitsSet", "oddBitsSet", ""}, + //{xor, "innerBitsSet", "evenBitsSet", ""}, + // + {xor, "outerBitsSet", "empty", "outerBitsSet"}, + {xor, "outerBitsSet", "full", "innerBitsSet"}, + {xor, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {xor, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {xor, "outerBitsSet", "firstBitUnset", "lastBitUnset"}, + {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, + {xor, "outerBitsSet", "innerBitsSet", "full"}, + {xor, "outerBitsSet", "outerBitsSet", "empty"}, + //{xor, "outerBitsSet", "oddBitsSet", ""}, + //{xor, "outerBitsSet", "evenBitsSet", ""}, + // + {xor, "oddBitsSet", "empty", "oddBitsSet"}, + {xor, "oddBitsSet", "full", "evenBitsSet"}, + //{xor, "oddBitsSet", "firstBitSet", ""}, + //{xor, "oddBitsSet", "lastBitSet", ""}, + //{xor, "oddBitsSet", "firstBitUnset", ""}, + //{xor, "oddBitsSet", "lastBitUnset", ""}, + //{xor, "oddBitsSet", "innerBitsSet", ""}, + //{xor, "oddBitsSet", "outerBitsSet", ""}, + {xor, "oddBitsSet", "oddBitsSet", "empty"}, + {xor, "oddBitsSet", "evenBitsSet", "full"}, + // + {xor, "evenBitsSet", "empty", "evenBitsSet"}, + {xor, "evenBitsSet", "full", "oddBitsSet"}, + //{xor, "evenBitsSet", "firstBitSet", ""}, + //{xor, "evenBitsSet", "lastBitSet", ""}, + //{xor, "evenBitsSet", "firstBitUnset", ""}, + //{xor, "evenBitsSet", "lastBitUnset", ""}, + //{xor, "evenBitsSet", "innerBitsSet", ""}, + //{xor, "evenBitsSet", "outerBitsSet", ""}, + {xor, "evenBitsSet", "oddBitsSet", "full"}, + {xor, "evenBitsSet", "evenBitsSet", "empty"}, + } + for _, testOp := range testOps { + for _, x := range containerTypes { + for _, y := range containerTypes { + desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) + ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y]) + exp := testOp.exp + + // Convert to all container types and check result. + for _, ct := range containerTypes { + clone := ret.clone() + if ct == ContainerArray { + if clone.isBitmap() { + clone.bitmapToArray() + } else if clone.isRun() { + clone.runToArray() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + // Because xorRunRun resulting in an empty container returns an array container with a + // nil slice array, then we need to check len() on array first (look for 0). + if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { + t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) + } + } else if ct == ContainerBitmap { + if clone.isArray() { + clone.arrayToBitmap() + } else if clone.isRun() { + clone.runToBitmap() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { + t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) + } + } else if ct == ContainerRun { + if clone.isArray() { + clone.arrayToRun() + } else if clone.isBitmap() { + clone.bitmapToRun() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) { + t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs) + } + } + } + } + } } - return bitmap } diff --git a/server.go b/server.go index a3143cbf3..b8ee6f70a 100644 --- a/server.go +++ b/server.go @@ -15,6 +15,7 @@ package pilosa import ( + "context" "crypto/tls" "errors" "fmt" diff --git a/server/default.go b/server/default.go index 6e9726afe..a7131c82a 100644 --- a/server/default.go +++ b/server/default.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // +build !release // // This file sets defaults to be overridden by release.go diff --git a/server/release.go b/server/release.go index de5b4a4d0..54d00b244 100644 --- a/server/release.go +++ b/server/release.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // +build release // // This file sets release-specific variables. diff --git a/server/server.go b/server/server.go index ef07ee381..19a08f982 100644 --- a/server/server.go +++ b/server/server.go @@ -16,6 +16,7 @@ // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the // objects that Pilosa needs. + package server import ( diff --git a/stats_test.go b/stats_test.go index 07254a702..93d7d9bb1 100644 --- a/stats_test.go +++ b/stats_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/test/attr.go b/test/attr.go index 620848084..9363011b1 100644 --- a/test/attr.go +++ b/test/attr.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/client.go b/test/client.go index 5eac8cb79..9e391e88b 100644 --- a/test/client.go +++ b/test/client.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/cluster.go b/test/cluster.go index 55f5ad074..f5e0dc7a9 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/executor.go b/test/executor.go index 85445719d..afe244ad7 100644 --- a/test/executor.go +++ b/test/executor.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/fragment.go b/test/fragment.go index 36ebba5bf..b208e7a3a 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/frame.go b/test/frame.go index 6234c802e..e107b7d85 100644 --- a/test/frame.go +++ b/test/frame.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/handler.go b/test/handler.go index 4859443c6..ea44bef9a 100644 --- a/test/handler.go +++ b/test/handler.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/holder.go b/test/holder.go index 4cd0642e4..c9d21d2f2 100644 --- a/test/holder.go +++ b/test/holder.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/index.go b/test/index.go index a751b3af6..5354c85f9 100644 --- a/test/index.go +++ b/test/index.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/pilosa.go b/test/pilosa.go index 7083b07b7..3d741af76 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/pilosa_test.go b/test/pilosa_test.go index c4842bf8b..583a244c0 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test_test import ( diff --git a/test/test.go b/test/test.go index cf98c76b8..c83b657c0 100644 --- a/test/test.go +++ b/test/test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import "flag" diff --git a/version.go b/version.go index 3c7d19ca3..3e148ea7b 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa var Version = "v0.0.0"