mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
Merge branch 'master' into cluster-resize
This commit is contained in:
Travis Turner
commit
62cd22f017
24 changed files with 304 additions and 261 deletions
47
CHANGELOG.md
47
CHANGELOG.md
|
|
@ -5,6 +5,53 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/).
|
||||
|
||||
## [0.8.8] - 2018-02-19
|
||||
|
||||
This version contains 1 contribution from 2 contributors. There are 4 files changed, 1,153 insertions, and 618 deletions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118))
|
||||
|
||||
## [0.8.7] - 2018-02-12
|
||||
|
||||
This version contains 1 contribution from 1 contributors. There are 2 files changed, 84 insertions, and 4 deletions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111))
|
||||
|
||||
## [0.8.6] - 2018-02-09
|
||||
|
||||
This version contains 2 contributions from 2 contributors. There are 3 files changed, 171 insertions, and 6 deletions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106)
|
||||
- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083)
|
||||
|
||||
## [0.8.5] - 2018-01-18
|
||||
|
||||
This version contains 1 contribution from 1 contributor. There is 1 file changed, 1 insertion, and 0 deletions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061))
|
||||
|
||||
## [0.8.4] - 2018-01-10
|
||||
|
||||
This version contains 4 contributions from 3 contributors. There are 17 files changed, 974 insertions, and 221 deletions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038))
|
||||
- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033))
|
||||
|
||||
### Performance
|
||||
|
||||
- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046))
|
||||
- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042))
|
||||
|
||||
## [0.8.3] - 2017-12-12
|
||||
|
||||
This version contains 1 contribution from 1 contributor. There are 2 files changed, 59 insertions, and 42 deletions.
|
||||
|
|
|
|||
|
|
@ -2,30 +2,83 @@
|
|||
|
||||
## Reporting a bug
|
||||
|
||||
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]
|
||||
If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1].
|
||||
|
||||
## Submitting a feature request
|
||||
|
||||
Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4]
|
||||
Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4]
|
||||
which starts by [opening a new issue][1] that describes the new feature proposal.
|
||||
|
||||
## Submitting code changes
|
||||
## Making code contributions
|
||||
|
||||
Before you start working on new features, you should [open a new issue][1] to let others know what
|
||||
you're doing before you start working, otherwise you run the risk of duplicating effort. This also
|
||||
you're doing, otherwise you run the risk of duplicating effort. This also
|
||||
gives others an opportunity to provide input for your feature.
|
||||
|
||||
If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
|
||||
|
||||
- Fork the [Pilosa repository][2] and then clone your fork:
|
||||
|
||||
```shell
|
||||
git clone git@github.com:<your-name>/pilosa.git
|
||||
### Development Environment
|
||||
|
||||
- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../.travis.yml) for the most up-to-date information.
|
||||
|
||||
- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
|
||||
|
||||
- Fork the [Pilosa repository][2] to your own account.
|
||||
|
||||
- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
|
||||
|
||||
```sh
|
||||
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
|
||||
git clone git@github.com:${USER}/pilosa.git
|
||||
```
|
||||
|
||||
- `cd` to your pilosa directory:
|
||||
|
||||
```sh
|
||||
cd ${GOPATH}/src/github.com/pilosa/pilosa
|
||||
```
|
||||
|
||||
- [Install](https://github.com/golang/dep/#installation) `dep` to manage dependencies:
|
||||
|
||||
```sh
|
||||
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
|
||||
```
|
||||
|
||||
- Install Pilosa command line tools:
|
||||
|
||||
```sh
|
||||
make install
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```
|
||||
dep ensure && go install github.com/pilosa/pilosa/cmd/...
|
||||
```
|
||||
|
||||
Running `pilosa` should now run a Pilosa instance.
|
||||
|
||||
- In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
|
||||
|
||||
```sh
|
||||
cd ${GOPATH}/src/github.com/pilosa/pilosa
|
||||
git remote add upstream git@github.com:pilosa/pilosa.git
|
||||
```
|
||||
|
||||
### Submitting code changes
|
||||
|
||||
- Before starting to work on a task, sync your branch with the upstream:
|
||||
|
||||
```sh
|
||||
git fetch upstream
|
||||
git checkout master
|
||||
git merge upstream/master
|
||||
```
|
||||
|
||||
- Create a local feature branch:
|
||||
|
||||
```shell
|
||||
```sh
|
||||
git checkout -b something-amazing
|
||||
```
|
||||
|
||||
|
|
@ -33,13 +86,13 @@ If you want to help but you aren't sure where to start, check out our [github la
|
|||
|
||||
- Make sure that you've written tests for your new feature, and then run the tests:
|
||||
|
||||
```shell
|
||||
```sh
|
||||
make test
|
||||
```
|
||||
|
||||
- Verify that your pull request is applied to the latest version of code on github:
|
||||
|
||||
```shell
|
||||
```sh
|
||||
git remote add upstream git@github.com:pilosa/pilosa.git
|
||||
git fetch upstream
|
||||
git rebase -i upstream/master
|
||||
|
|
@ -47,7 +100,7 @@ If you want to help but you aren't sure where to start, check out our [github la
|
|||
|
||||
- Push to your fork:
|
||||
|
||||
```shell
|
||||
```sh
|
||||
git push -u <yourfork> something-amazing
|
||||
```
|
||||
|
||||
|
|
@ -59,4 +112,4 @@ If you want to help but you aren't sure where to start, check out our [github la
|
|||
[3]: https://github.com/pilosa/pilosa/compare/
|
||||
[4]: https://github.com/pilosa/general/blob/master/proposal.md
|
||||
[5]: https://github.com/pilosa/pilosa/issues
|
||||
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer
|
||||
[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
|
||||
Development Environment
|
||||
=======================
|
||||
|
||||
Install Go versions 1.6.2+ or 1.7 for your platform.
|
||||
|
||||
Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private.
|
||||
|
||||
Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`.
|
||||
|
||||
Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
|
||||
|
||||
```sh
|
||||
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
|
||||
git clone git@github.com:${USER}/pilosa.git
|
||||
```
|
||||
|
||||
`cd` to your pilosa directory:
|
||||
|
||||
```sh
|
||||
cd ${GOPATH}/src/github.com/pilosa/pilosa
|
||||
```
|
||||
|
||||
Install `dep` to manage dependencies:
|
||||
|
||||
```sh
|
||||
go get -u github.com/golang/dep/cmd/dep
|
||||
```
|
||||
|
||||
Install Pilosa command line tools:
|
||||
|
||||
```sh
|
||||
make install
|
||||
# or:
|
||||
# dep ensure && go install github.com/pilosa/pilosa/cmd/...
|
||||
```
|
||||
|
||||
Running `pilosa` should now run a Pilosa instance.
|
||||
|
||||
In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
|
||||
|
||||
```sh
|
||||
cd ${GOPATH}/src/github.com/pilosa/pilosa
|
||||
git remote add upstream git@github.com:pilosa/pilosa.git
|
||||
```
|
||||
|
||||
Before starting to work on a task, sync your branch with the upstream:
|
||||
|
||||
```sh
|
||||
git fetch upstream
|
||||
git checkout master
|
||||
git merge upstream/master
|
||||
```
|
||||
|
||||
Create a branch for the task:
|
||||
|
||||
```sh
|
||||
git checkout -b a-branch-for-the-task
|
||||
```
|
||||
|
||||
Update the code in the branch, and commit it.
|
||||
|
||||
Push it to your own repo:
|
||||
|
||||
```sh
|
||||
git push --set-upstream origin a-branch-for-the-task
|
||||
```
|
||||
|
||||
All left to do is creating a pull request on github.com.
|
||||
|
|
@ -87,7 +87,7 @@ func TestImportCommand_Run(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure that the ImportValue path runs (note: we have specifed a value
|
||||
// Ensure that the ImportValue path runs (note: we have specified a value
|
||||
// for cm.Field. Because the handler doesn't return errors (it sends them
|
||||
// to the logger), we don't get an error returned at `cm.Run()` even though
|
||||
// we haven't setup frame `f` to be RangeEnabled.
|
||||
|
|
|
|||
5
docs/README.md
Normal file
5
docs/README.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/).
|
||||
|
||||
Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search.
|
||||
|
||||
Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request!
|
||||
|
|
@ -65,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
|
|||
```
|
||||
|
||||
<div class="note">
|
||||
<p>Note that you must first create a frame with Range Encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
|
||||
<p>Note that you must first create a frame with range-encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
|
||||
</div>
|
||||
|
||||
#### Exporting
|
||||
|
|
@ -178,17 +178,17 @@ Note: This will only work when the replication factor is >= 2
|
|||
|
||||
#### Copying data files manually
|
||||
|
||||
- To accomplish this goal you will 1st need:
|
||||
- List of all Indexes on your cluster
|
||||
- List of all frames in your Indexes
|
||||
- Max slice per Index, listed in the /status endpoint
|
||||
- To accomplish this you will first need:
|
||||
- List of all indexes on your cluster
|
||||
- List of all frames in your indexes
|
||||
- Max slice per index, listed in the /status endpoint
|
||||
- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
|
||||
- Using the list of slices owned by this node you will then need to manually:
|
||||
- setup a directory structure similar to the other nodes with a path for each Index/Frame
|
||||
- copy each owned slice for an existing node to this new node
|
||||
- Modify the cluster config file to replace the previous node address with the new node address.
|
||||
- Restart the cluster
|
||||
- Wait for the 1st sync (10 minutes) to validate Index connections
|
||||
- Wait for the first sync (10 minutes) to validate Index connections
|
||||
|
||||
### Diagnostics
|
||||
|
||||
|
|
@ -216,9 +216,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman
|
|||
Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default.
|
||||
The metrics configuration options are:
|
||||
|
||||
- [Host](../configuration#metrics-host): specify host that receives metric events
|
||||
- [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics
|
||||
- [Service](../configuration#metrics-service): declare type StatsD or Expvar
|
||||
- [Host](../configuration/#metric-host): specify host that receives metric events
|
||||
- [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
|
||||
- [Service](../configuration/#metric-service): declare type StatsD or Expvar
|
||||
|
||||
#### Tags
|
||||
StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ In order to send protobuf binaries in the request and response, set `Content-Typ
|
|||
|
||||
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
|
||||
|
||||
The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
|
||||
The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
|
||||
|
||||
``` request
|
||||
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
|
||||
|
|
@ -102,14 +102,15 @@ Creates a frame in the given index with the given name.
|
|||
|
||||
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
|
||||
|
||||
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
|
||||
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
|
||||
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
|
||||
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
|
||||
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
|
||||
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
|
||||
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
|
||||
* `rangeEnabled` (boolean): Enables range-encoded fields in this frame.
|
||||
* `fields` (array): List of range-encoded fields.
|
||||
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
|
||||
|
||||
Each individual `field` contains the following:
|
||||
|
||||
* `name` (string): Field name.
|
||||
* `type` (string): Field type, currently only "int" is supported.
|
||||
* `min` (int): Minimum value allowed for this field.
|
||||
|
|
@ -199,15 +200,19 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \
|
|||
|
||||
### Create input definition
|
||||
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of v0.9.
|
||||
</div>
|
||||
|
||||
`POST /index/<index-name>/input-definition/<input-definition-name>`
|
||||
|
||||
Creates an input definition in the given index with the given name.
|
||||
|
||||
The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options:
|
||||
|
||||
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
|
||||
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
|
||||
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
|
||||
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
|
||||
* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`.
|
||||
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
|
||||
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
|
||||
|
||||
The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa.
|
||||
|
|
@ -263,6 +268,10 @@ curl localhost:10101/index/user/input-definition/stargazer-input \
|
|||
|
||||
### Get input definition
|
||||
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of v0.9.
|
||||
</div>
|
||||
|
||||
`GET /index/<index-name>/input-definition/<input-definition-name>`
|
||||
|
||||
Returns the given input definition as JSON.
|
||||
|
|
@ -276,6 +285,10 @@ curl -XGET localhost:10101/index/user/input-definition/stargazer-input
|
|||
|
||||
### Remove input definition
|
||||
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of v0.9.
|
||||
</div>
|
||||
|
||||
`DELETE /index/<index-name>/input-definition/<input-definition-name>`
|
||||
|
||||
Removes the given input definition.
|
||||
|
|
@ -289,6 +302,10 @@ curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input
|
|||
|
||||
### Process input data
|
||||
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of v0.9.
|
||||
</div>
|
||||
|
||||
`POST /index/<index-name>/input/<input-definition-name>`
|
||||
|
||||
Processes the JSON payload using the given input definition.
|
||||
|
|
|
|||
|
|
@ -18,5 +18,8 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B
|
|||
* After the container storage section is an operation log, of unspecified length.
|
||||
|
||||
|
||||
*Pilosa Roaring storage format diagram*
|
||||
|
||||
All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.
|
||||
|
||||
Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ This section contains example code for client libraries in several languages. Pl
|
|||
|
||||
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
|
|
@ -96,9 +96,9 @@ func main() {
|
|||
|
||||
### Python
|
||||
|
||||
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) for more information and installation instructions.
|
||||
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
|
|
@ -171,7 +171,7 @@ client.query(stargazer.setbit(99999, 77777))
|
|||
|
||||
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
|
||||
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`.
|
||||
|
||||
Error handling has been omitted in the example below for brevity.
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ Pilosa lays out data first in rows, so queries which get all the set bits in one
|
|||
|
||||
Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but if you try to set a bit with column ID 2^63, bad things will start to happen.
|
||||
|
||||
|
||||
|
||||
*Basic data model diagram*
|
||||
|
||||
### Index
|
||||
|
||||
|
|
@ -51,16 +52,18 @@ Row attributes are namespaced at the Frame level.
|
|||
Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
|
||||
|
||||
|
||||
*Ranked frame diagram*
|
||||
|
||||
#### LRU
|
||||
|
||||
The LRU cache maintains the most recently accessed Rows.
|
||||
|
||||
|
||||
*LRU frame diagram*
|
||||
|
||||
### Time Quantum
|
||||
|
||||
Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
|
||||
Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
|
||||
|
||||
### Attribute
|
||||
|
||||
|
|
@ -92,6 +95,7 @@ SetBit(frame="A", rowID=19, columnID=5)
|
|||
```
|
||||
|
||||
|
||||
*Inverse frame diagram*
|
||||
|
||||
#### Time Quantums
|
||||
|
||||
|
|
@ -103,23 +107,27 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
|
|||
```
|
||||
|
||||
|
||||
*Time quantum frame diagram*
|
||||
|
||||
#### BSI Range-Encoding
|
||||
|
||||
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded
|
||||
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate, aggregate, and range queries on these BSI integers.
|
||||
bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers.
|
||||
|
||||
Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
|
||||
|
||||
For example, the following `SetFieldValue()` queries will result in the data described in the illustration below:
|
||||
|
||||
```
|
||||
SetFieldValue(col=1, frame="A", field0=1)
|
||||
SetFieldValue(col=2, frame="A", field0=2)
|
||||
SetFieldValue(col=3, frame="A", field0=3)
|
||||
SetFieldValue(col=4, frame="A", field0=7)
|
||||
SetFieldValue(col=2, frame="A", field1=1)
|
||||
SetFieldValue(col=3, frame="A", field1=6)
|
||||
SetFieldValue(columnID=1, frame="A", field0=1)
|
||||
SetFieldValue(columnID=2, frame="A", field0=2)
|
||||
SetFieldValue(columnID=3, frame="A", field0=3)
|
||||
SetFieldValue(columnID=4, frame="A", field0=7)
|
||||
SetFieldValue(columnID=2, frame="A", field1=1)
|
||||
SetFieldValue(columnID=3, frame="A", field1=6)
|
||||
```
|
||||
|
||||
|
||||
|
||||
*BSI frame diagram*
|
||||
|
||||
Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ New York City released an extremely detailed data set of over 1 billion taxi rid
|
|||
|
||||
Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
|
||||
|
||||
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
|
||||
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
|
||||
|
||||
After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ durm := pdk.CustomMapper{
|
|||
|
||||
#### Import process
|
||||
|
||||
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
|
||||
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk/) for more details on this process.
|
||||
|
||||
#### Queries
|
||||
|
||||
|
|
@ -214,7 +214,7 @@ T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
|
|||
|
||||
A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
|
||||
|
||||
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
|
||||
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a [Github repository](https://github.com/pilosa/chem-usecase).
|
||||
|
||||
#### Data model
|
||||
|
||||
|
|
@ -263,7 +263,7 @@ python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
|
|||
```
|
||||
|
||||
|
||||
First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
|
||||
First, follow the instruction in the [getting started](../getting-started/) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
|
||||
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
|
||||
```
|
||||
curl localhost:10101/index/mole \
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ Any HTTP tool can be used to interact with the Pilosa server. The examples in th
|
|||
|
||||
### Starting Pilosa
|
||||
|
||||
Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa.
|
||||
Follow the steps in the [Install](../installation/) document to install Pilosa.
|
||||
Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`):
|
||||
```
|
||||
pilosa server
|
||||
|
|
@ -42,7 +42,7 @@ curl localhost:10101/status
|
|||
|
||||
In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project.
|
||||
|
||||
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation.
|
||||
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation.
|
||||
|
||||
#### Create the Schema
|
||||
|
||||
|
|
@ -87,7 +87,7 @@ curl localhost:10101/index/repository/frame/language \
|
|||
|
||||
#### Import Data From CSV Files
|
||||
|
||||
If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo.
|
||||
Download the `stargazer.csv` and `language.csv` files here:
|
||||
|
||||
```
|
||||
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
|
||||
|
|
@ -112,7 +112,12 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv
|
|||
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
|
||||
|
||||
### Input Definition
|
||||
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
|
||||
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of v0.9.
|
||||
</div>
|
||||
|
||||
Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data.
|
||||
|
||||
#### Make Some Queries
|
||||
|
||||
|
|
|
|||
|
|
@ -6,53 +6,62 @@ nav = []
|
|||
|
||||
## Glossary
|
||||
|
||||
<strong id="index">Index:</strong> Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes.
|
||||
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
|
||||
|
||||
<strong id="column">Column:</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index.
|
||||
<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
|
||||
|
||||
<strong id="row">Row:</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index.
|
||||
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column).
|
||||
|
||||
<strong id="bit">Bit:</strong> A bit is the intersection of a Row and Column.
|
||||
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap.
|
||||
|
||||
<strong id="bitmap">Bitmap:</strong> The on-disk and in-memory representation of a Row.
|
||||
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding)</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries.
|
||||
|
||||
<strong id="roaring-bitmap">Roaring Bitmap:</strong> [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses.
|
||||
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
|
||||
|
||||
<strong id="attribute">Attribute:</strong> Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store.
|
||||
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index).
|
||||
|
||||
<strong id="pql">PQL:</strong> Pilosa Query Language
|
||||
<strong id="field">[Field](../data-model/#bsi-range-encoding):</strong> A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries.
|
||||
|
||||
<strong id="index">Index:</strong> The Index represents a data namespace.
|
||||
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index).
|
||||
|
||||
<strong id="frame">Frame:</strong> Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame.
|
||||
<strong id="frame">[Frame](../data-model/#frame):</strong> Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame.
|
||||
|
||||
<strong id="view">View:</strong> Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
|
||||
<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes.
|
||||
|
||||
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a frame and slice in an index.
|
||||
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
|
||||
|
||||
<strong id="slice">Slice:</strong> Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash.
|
||||
<strong id="maxslice">MaxSlice:</strong> The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries.
|
||||
|
||||
<strong id="slicewidth">SliceWidth:</strong> This is the default number of columns in a slice.
|
||||
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
|
||||
|
||||
<strong id="maxslice">MaxSlice:</strong> The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries.
|
||||
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
|
||||
|
||||
<strong id="anti-entropy">Anti-entropy:</strong> A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies.
|
||||
<strong id="pql">[PQL](../query-language/):</strong> Pilosa Query Language.
|
||||
|
||||
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a cluster.
|
||||
<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
|
||||
|
||||
<strong id="cluster">Cluster:</strong> A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
|
||||
<strong id="range">[Range](../query-language/#range-queries):</strong>: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
|
||||
|
||||
<strong id="topn">TopN:</strong> Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row.
|
||||
<strong id="range-bsi">[Range (BSI)](../query-language/#range-bsi):</strong>: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
|
||||
|
||||
<strong id="tanimoto">Tanimoto:</strong> Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity.
|
||||
<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
|
||||
|
||||
<strong id="protobuf">Protobuf:</strong>: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
|
||||
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
|
||||
|
||||
<strong id="toml">TOML:</strong> We use [TOML](https://github.com/toml-lang/toml) for our configuration file format.
|
||||
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap).
|
||||
|
||||
<strong id="jump-consistent-hash">Jump Consistent Hash:</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
|
||||
https://arxiv.org/pdf/1406.2294v1.pdf
|
||||
<strong id="slice">[Slice](../data-model/#slice):</strong> [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
|
||||
|
||||
<strong id="partition">Partition:</strong> The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography.
|
||||
<strong id="slicewidth">SliceWidth:</strong> This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
|
||||
|
||||
<strong id="replica">Replica:</strong> A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made).
|
||||
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field).
|
||||
|
||||
<strong id="tanimoto">[Tanimoto](../examples/#chemical-similarity-search):</strong> Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|).
|
||||
|
||||
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for time [Range](#range) queries.
|
||||
|
||||
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
|
||||
|
||||
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame).
|
||||
|
||||
<strong id="view">[View](../data-model/#view):</strong> Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
|
||||
|
|
|
|||
|
|
@ -1,64 +1,11 @@
|
|||
+++
|
||||
title = "Input Definition"
|
||||
weight = 8
|
||||
nav = [
|
||||
"Create the Schema",
|
||||
"Import Data",
|
||||
]
|
||||
+++
|
||||
|
||||
## Input Definition
|
||||
This document builds on the data import concepts introduced in [Getting Started](../getting-started/).
|
||||
Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data.
|
||||
|
||||
### Create the Schema
|
||||
|
||||
Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations.
|
||||
|
||||
Before creating a schema, let's create the repository index first:
|
||||
|
||||
```
|
||||
curl localhost:10101/index/repository -X POST
|
||||
```
|
||||
The sample input definition schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started) in the `input_definition.json` file. Download it using:
|
||||
```
|
||||
curl -OL https://github.com/pilosa/getting-started/raw/master/input_definition.json
|
||||
```
|
||||
|
||||
Run the following to create the input definition:
|
||||
```
|
||||
curl localhost:10101/index/repository/input-definition/stargazer -d @input_definition.json
|
||||
```
|
||||
|
||||
Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition.
|
||||
We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination:
|
||||
|
||||
- value-to-row: The value for this field is used as the `rowID`.
|
||||
- single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type.
|
||||
- mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type.
|
||||
- set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame
|
||||
|
||||
### Import Data
|
||||
|
||||
The sample data for the input definition we created above is in the `json_input.json` file at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download it using:
|
||||
```
|
||||
curl -OL https://github.com/pilosa/getting-started/raw/master/json_input.json
|
||||
```
|
||||
|
||||
Then run the following to import it:
|
||||
```
|
||||
curl localhost:10101/index/repository/input/stargazer -d @json_input.json
|
||||
```
|
||||
|
||||
As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID.
|
||||
The data input above is equivalent to the following `SetBit()` operations:
|
||||
|
||||
```
|
||||
curl localhost:10101/index/repository/query \
|
||||
-X POST \
|
||||
-d 'SetBit(frame="stargazer", columnID=91720568, rowID=513114)
|
||||
SetBit(frame="stargazer", columnID=91720568, rowID=513114, timestamp="2017-05-18T20:40")
|
||||
SetBit(frame="language", columnID=91720568, rowID=5)
|
||||
SetBit(frame="language", columnID=95122322, rowID=17)
|
||||
'
|
||||
```
|
||||
<div class="warning">
|
||||
Input definition is deprecated as of Pilosa v0.9.<br/>
|
||||
<br/>
|
||||
The previous version of this page is still available <a href="https://www.pilosa.com/docs/v0.8/input-definition">here</a>.
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -136,15 +136,16 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
|
||||
1. Install the prerequisites:
|
||||
|
||||
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
|
||||
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
|
||||
* [Git](https://git-scm.com/)
|
||||
|
||||
2. Clone the repo:
|
||||
```
|
||||
go get -d github.com/pilosa/pilosa
|
||||
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
|
||||
git clone https://github.com/pilosa/pilosa.git
|
||||
```
|
||||
|
||||
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice web console into Pilosa):
|
||||
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
|
||||
```
|
||||
cd $GOPATH/src/github.com/pilosa/pilosa
|
||||
make generate-statik
|
||||
|
|
@ -203,7 +204,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
|
|||
docker version
|
||||
```
|
||||
|
||||
If you don't see the server listed, start the Docker application.
|
||||
If you don't see the server listed, start the Docker application.
|
||||
|
||||
3. Pull the official Pilosa image from Docker Hub:
|
||||
```
|
||||
|
|
@ -289,17 +290,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
|
|||
|
||||
1. Install the prerequisites:
|
||||
|
||||
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
|
||||
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
|
||||
* [Git](https://git-scm.com/)
|
||||
|
||||
2. Clone the repo:
|
||||
```
|
||||
go get -d github.com/pilosa/pilosa
|
||||
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
|
||||
git clone https://github.com/pilosa/pilosa.git
|
||||
```
|
||||
|
||||
3. Build the Pilosa repo:
|
||||
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
|
||||
```
|
||||
cd $GOPATH/src/github.com/pilosa/pilosa
|
||||
make generate-statik
|
||||
make install
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ It is designed primarly for speed and horizontal scalability. If you have data w
|
|||
|
||||
"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
|
||||
|
||||
Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.
|
||||
Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ nav = [
|
|||
|
||||
### Overview
|
||||
|
||||
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
|
||||
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index](../glossary/#index) and are passed to Pilosa through the `/index/INDEX_NAME/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
|
||||
|
||||
```
|
||||
{"results":[...]}
|
||||
|
|
@ -31,7 +31,7 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch
|
|||
|
||||
##### Examples
|
||||
|
||||
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data.
|
||||
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data.
|
||||
|
||||
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would:
|
||||
``` request
|
||||
|
|
@ -45,7 +45,7 @@ curl localhost:10101/index/repository/query \
|
|||
|
||||
#### Arguments and Types
|
||||
|
||||
* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
|
||||
* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
|
||||
* `ROW_LABEL` The default row label is `rowID`, changing the default is deprecated.
|
||||
* `COL_LABEL` The default column label is `columnID`, changing the default is deprecated.
|
||||
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
|
||||
|
|
@ -118,7 +118,7 @@ SetRowAttrs queries always return `null` upon success.
|
|||
SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true)
|
||||
```
|
||||
|
||||
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
|
||||
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
|
||||
|
||||
```
|
||||
SetRowAttrs(frame="stargazer", rowID=10, username=null)
|
||||
|
|
@ -150,7 +150,7 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu
|
|||
SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true)
|
||||
```
|
||||
|
||||
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", columnID=10)`.
|
||||
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`.
|
||||
|
||||
```
|
||||
SetColumnAttrs(columnID=10, url=null)
|
||||
|
|
@ -187,6 +187,30 @@ ClearBit(frame="stargazer", columnID=10, rowID=1)
|
|||
Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame.
|
||||
|
||||
|
||||
#### SetFieldValue
|
||||
|
||||
**Spec:**
|
||||
|
||||
```
|
||||
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
|
||||
```
|
||||
|
||||
**Description:**
|
||||
|
||||
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
|
||||
|
||||
**Result Type:** null
|
||||
|
||||
SetFieldValue returns `null` upon success.
|
||||
|
||||
**Examples:**
|
||||
|
||||
Set the number of pull requests of repository 10.
|
||||
```
|
||||
SetFieldValue(columnID=10, frame="stats", pullrequests=2)
|
||||
```
|
||||
|
||||
|
||||
### Read Operations
|
||||
|
||||
#### Bitmap
|
||||
|
|
@ -380,6 +404,7 @@ have the attribute specified by `field` with one of the values specified in
|
|||
**Result Type:** array of key/count objects
|
||||
|
||||
**Caveats:**
|
||||
|
||||
* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order.
|
||||
* Frames with cache type lru will maintain an LRU (Least Recently Used) cache, thus a TopN() query on this type of frame will return bitmaps sorted in order of most recently set bit.
|
||||
* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN() queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance.
|
||||
|
|
@ -472,7 +497,7 @@ Returns bits that are true for the comparison operator.
|
|||
**Examples:**
|
||||
|
||||
In our source data, commitactivity was counted over the last year.
|
||||
The following greater-than Range query returns all repositories having more than 100 commits.
|
||||
The following greater-than `Range` query returns all repositories having more than 100 commits.
|
||||
|
||||
```
|
||||
Range(frame="stats", commitactivity > 100)
|
||||
|
|
@ -512,7 +537,7 @@ Sum([BITMAP_CALL], <frame=STRING>, <field=STRING>)
|
|||
|
||||
**Description:**
|
||||
|
||||
Returns the count and computed sum of all bitmap encoded integer values across the `field` in this `frame`. The optional Bitmap call filters the bits used in this computation.
|
||||
Returns the count and computed sum of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns.
|
||||
|
||||
**Result Type:** object with the computed sum and count of the bitmap field.
|
||||
|
||||
|
|
@ -526,27 +551,3 @@ Sum(frame="stats", field="diskusage")
|
|||
Return `{"sum":10,"count":3}`
|
||||
|
||||
* Result is the size of all repositories in kilobytes, plus the number of repositories.
|
||||
|
||||
|
||||
#### SetFieldValue
|
||||
|
||||
**Spec:**
|
||||
|
||||
```
|
||||
SetFieldValue(<COL_LABEL=UINT>, <frame=STRING>, <FIELD_NAME=INT>)
|
||||
```
|
||||
|
||||
**Description:**
|
||||
|
||||
`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`.
|
||||
|
||||
**Result Type:** null
|
||||
|
||||
SetFieldValue returns `null` upon success.
|
||||
|
||||
**Examples:**
|
||||
|
||||
Set the number of pull requests of repository 10.
|
||||
```
|
||||
SetFieldValue(col=10, frame="stats", pullrequests=2)
|
||||
```
|
||||
|
|
|
|||
|
|
@ -10,6 +10,16 @@ nav = [
|
|||
|
||||
## Tutorials
|
||||
|
||||
<div class="note">
|
||||
<!-- this is html because there is a problem putting a list inside a shortcode -->
|
||||
Some of our tutorials work better as standalone repos, since you can <code>git clone</code> the instructions, code, and data all at once. Officially supported tutorials are listed here.<br />
|
||||
<br />
|
||||
<ul>
|
||||
<li><a href="https://github.com/pilosa/cosmosa">Run Pilosa with Microsoft's Azure Cosmos DB</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
|
||||
### Setting Up a Secure Cluster
|
||||
|
||||
#### Introduction
|
||||
|
|
@ -20,7 +30,7 @@ This tutorial assumes that you are using a UNIX-like system, such as Linux or Ma
|
|||
|
||||
#### Installing Pilosa and Creating the Directory Structure
|
||||
|
||||
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](https://www.pilosa.com/docs/latest/installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](https://www.pilosa.com/docs/latest/installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](https://www.pilosa.com/docs/latest/installation/#build-from-source).
|
||||
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
|
||||
|
||||
After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
|
||||
```
|
||||
|
|
@ -45,6 +55,7 @@ openssl req -x509 -newkey rsa:2048 -keyout pilosa.local.key -out pilosa.local.cr
|
|||
```
|
||||
|
||||
The command above creates two files in the current directory:
|
||||
|
||||
* `pilosa.local.crt` is the SSL certificate.
|
||||
* `pilosa.local.key` is the private key file which must be kept as secret.
|
||||
|
||||
|
|
@ -126,9 +137,10 @@ key = "pilosa.local.gossip32"
|
|||
```
|
||||
|
||||
Here is some explanation of the configuration items:
|
||||
|
||||
* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
|
||||
* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
|
||||
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](https://www.pilosa.com/docs/latest/configuration/#cluster-hosts) for other settings.
|
||||
* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings.
|
||||
* `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers.
|
||||
* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before.
|
||||
|
||||
|
|
@ -235,7 +247,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ
|
|||
|
||||
#### Introduction
|
||||
|
||||
Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
|
||||
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
|
||||
|
||||
First, create an index called `patients`:
|
||||
``` request
|
||||
|
|
|
|||
|
|
@ -9,18 +9,20 @@ nav = [
|
|||
|
||||
## WebUI
|
||||
|
||||
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101)
|
||||
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
|
||||
|
||||
This can be used for constructing queries and viewing the cluster status.
|
||||
|
||||
### Console
|
||||
|
||||
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
|
||||
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
|
||||
|
||||
Each query's result will be displayed in the Output section along with the query time.
|
||||
|
||||
The Console will keep a record of each query and its result with the latest query on top.
|
||||
|
||||
|
||||
|
||||
*WebUI console screenshot*
|
||||
|
||||
In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
|
||||
|
||||
|
|
@ -30,9 +32,8 @@ In addition to standard PQL, the console supports a few special commands, prefix
|
|||
- `:create frame <framename>`
|
||||
- `:delete frame <framename>`
|
||||
|
||||
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference).
|
||||
Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame).
|
||||
|
||||
- `:create index <indexname> timeQuantum=YM`
|
||||
- `:create frame <framename> inverseEnabled=true cacheSize=10000`
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -800,7 +800,7 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
|
|||
return NewBitmap(), nil
|
||||
}
|
||||
|
||||
// LT[E] and GT[E] should return all not-null if selected range fully encompases valid field range.
|
||||
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range.
|
||||
if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) ||
|
||||
(cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) {
|
||||
return frag.FieldNotNull(field.BitDepth())
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
|
|||
t.Fatal("expected change")
|
||||
}
|
||||
|
||||
// Non-existant value.
|
||||
// Non-existent value.
|
||||
if value, exists, err := f.FieldValue(100, 11); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != 0 {
|
||||
|
|
|
|||
|
|
@ -1911,13 +1911,14 @@ func intersectionCountBitmapRun(a, b *container) (n int) {
|
|||
}
|
||||
|
||||
func intersectionCountArrayBitmap(a, b *container) (n int) {
|
||||
ln := len(b.bitmap)
|
||||
for _, val := range a.array {
|
||||
i := val >> 6
|
||||
if i >= uint16(len(b.bitmap)) {
|
||||
i := int(val >> 6)
|
||||
if i >= ln {
|
||||
break
|
||||
}
|
||||
off := val % 64
|
||||
n += int((b.bitmap[i] & (1 << off)) >> off)
|
||||
n += int(b.bitmap[i]>>off) & 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
|
@ -2645,7 +2646,7 @@ func differenceRunRun(a, b *container) *container {
|
|||
for apos < alen && bpos < blen {
|
||||
switch {
|
||||
case alast < bstart:
|
||||
// current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run
|
||||
// current A-run entirely precedes current B-run: keep full A-run, advance to next A-run
|
||||
output.runs = append(output.runs, interval16{start: astart, last: alast})
|
||||
apos++
|
||||
if apos < alen {
|
||||
|
|
@ -2653,7 +2654,7 @@ func differenceRunRun(a, b *container) *container {
|
|||
alast = a.runs[apos].last
|
||||
}
|
||||
case blast < astart:
|
||||
// current B-run entirely preceeds current A-run: advance to next B-run
|
||||
// current B-run entirely precedes current A-run: advance to next B-run
|
||||
bpos++
|
||||
if bpos < blen {
|
||||
bstart = b.runs[bpos].start
|
||||
|
|
|
|||
|
|
@ -291,7 +291,7 @@ func TestBitmap_Max(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure CountRange is correct even if rangekey is prior to inital container.
|
||||
// Ensure CountRange is correct even if rangekey is prior to initial container.
|
||||
func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
|
||||
s := uint64(2009 * 1048576)
|
||||
e := uint64(2010 * 1048576)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import (
|
|||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
// StatsD protocal wrapper using the DataDog library that added Tags to the StatsD protocal
|
||||
// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol
|
||||
// statsD defailt host is "127.0.0.1:8125"
|
||||
|
||||
const (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue