scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into metrics

Browse source 
Conflicts:
	cmd/server.go
	ctl/config.go
This commit is contained in:
Michael Baird 2017-05-23 10:07:44 -05:00
commit b0f8d5d8f9
35 changed files with 2629 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -3,3 +3,4 @@ default.etcd/
vendor
.protoc-gen-gofast
.DS_Store
build

3
Makefile
View file

@ -39,12 +39,13 @@ pilosa: vendor
crossbuild: vendor
mkdir -p build/pilosa-$(IDENTIFIER)
make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
cp {LICENSE,README.md} build/pilosa-$(IDENTIFIER)
cp LICENSE README.md build/pilosa-$(IDENTIFIER)
tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
release:
make crossbuild GOOS=linux GOARCH=amd64
make crossbuild GOOS=linux GOARCH=386
make crossbuild GOOS=darwin GOARCH=amd64
install: vendor

11
client.go
View file

@ -486,7 +486,16 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view s
tw := tar.NewWriter(w)
// Find the maximum number of slices.
maxSlices, err := c.MaxSliceByIndex(ctx)
var maxSlices map[string]uint64
var err error
if view == ViewStandard {
maxSlices, err = c.MaxSliceByIndex(ctx)
} else if view == ViewInverse {
maxSlices, err = c.MaxInverseSliceByIndex(ctx)
} else {
return ErrInvalidView
}
if err != nil {
return fmt.Errorf("slice n: %s", err)
}

81
client_test.go
View file

@ -333,6 +333,87 @@ func TestClient_BackupRestore(t *testing.T) {
}
}
// Ensure client backup and restore a frame with inverse view.
func TestClient_BackupInverseView(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
f.SetBit(100, 1)
f.SetBit(100, 2)
f.SetBit(100, 3)
f.SetBit(100, SliceWidth-1)
s := NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
c := MustNewClient(s.Host())
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
}
// backup returns error with invalid view
func TestClient_BackupInvalidView(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
s := NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
c := MustNewClient(s.Host())
// Backup from frame.
var buf bytes.Buffer
err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view")
if err != pilosa.ErrInvalidView {
t.Fatal(err)
}
}
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := MustOpenHolder()

6
cmd/backup.go
View file

@ -43,9 +43,9 @@ Backs up the view from across the cluster into a single file.
}
flags := backupCmd.Flags()
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup into.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.")
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.")
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
return backupCmd

12
cmd/config.go
View file

@ -22,18 +22,21 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
"github.com/pilosa/pilosa/server"
)
var Conf *ctl.ConfigCommand
func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr)
Server := server.NewCommand(stdin, stdout, stderr)
confCmd := &cobra.Command{
Use: "config",
Short: "Print the default configuration.",
Long: `config prints the default configuration to stdout
`,
Short: "Print the current configuration.",
Long: `config prints the current configuration to stdout`,
RunE: func(cmd *cobra.Command, args []string) error {
Conf.Config = Server.Config
if err := Conf.Run(context.Background()); err != nil {
return err
}
@ -41,6 +44,9 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
},
}
// Attach flags to the command.
ctl.BuildServerFlags(confCmd, Server)
return confCmd
}

49
cmd/generate_config.go Normal file
View file

@ -0,0 +1,49 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
)
var GenerateConf *ctl.GenerateConfigCommand
func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
confCmd := &cobra.Command{
Use: "generate-config",
Short: "Print the default configuration.",
Long: `generate-config prints the default configuration to stdout
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := GenerateConf.Run(context.Background()); err != nil {
return err
}
return nil
},
}
return confCmd
}
func init() {
subcommandFns["generate-config"] = NewGenerateConfigCommand
}

24
cmd/server.go
View file

@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/ctl"
"github.com/pilosa/pilosa/server"
)
@ -85,27 +85,9 @@ on the configured port.`,
return nil
},
}
flags := serveCmd.Flags()
flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.IntVarP(&Server.Config.MaxWritesPerRequest, "max-writes-per-request", "", Server.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.")
flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this?
flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.LongQueryTime), "long-query-time", "", 10*time.Second, "Threshold for logging long-running queries (0 to disable)")
flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.")
flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path")
flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.")
flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]")
flags.StringVarP(&Server.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&Server.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&Server.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
flags.StringVarP(&Server.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
flags.DurationVarP((*time.Duration)(&Server.Config.Metric.PollingInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
// Attach flags to the command.
ctl.BuildServerFlags(serveCmd, Server)
return serveCmd
}

4
config.go
View file

@ -110,3 +110,7 @@ func (d *Duration) UnmarshalText(text []byte) error {
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
}
func (d Duration) MarshalTOML() ([]byte, error) {
return []byte(d.String()), nil
}

34
ctl/config.go
View file

@ -18,14 +18,15 @@ import (
"context"
"fmt"
"io"
"strings"
toml "github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
)
// ConfigCommand represents a command for printing a default config.
type ConfigCommand struct {
*pilosa.CmdIO
Config *pilosa.Config
}
// NewConfigCommand returns a new instance of ConfigCommand.
@ -37,31 +38,10 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand
// Run prints out the default config.
func (cmd *ConfigCommand) Run(ctx context.Context) error {
fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
data-dir = "~/.pilosa"
bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
poll-interval = "2m0s"
replicas = 1
hosts = [
"localhost:10101",
]
[anti-entropy]
interval = "10m0s"
[metric]
service = "statsd"
host = "127.0.0.1:8125"
[profile]
cpu = ""
cpu-time = "30s"
[plugins]
path = ""
`)+"\n")
buf, err := toml.Marshal(*cmd.Config)
if err != nil {
return err
}
fmt.Fprintln(cmd.Stdout, string(buf))
return nil
}

8
ctl/export.go
View file

@ -79,7 +79,13 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Determine slice count.
maxSlices, err := client.MaxSliceByIndex(ctx)
var maxSlices map[string]uint64
if cmd.View == pilosa.ViewStandard {
maxSlices, err = client.MaxSliceByIndex(ctx)
} else if cmd.View == pilosa.ViewInverse {
maxSlices, err = client.MaxInverseSliceByIndex(ctx)
}
if err != nil {
return err
}

63
ctl/generate_config.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"fmt"
"io"
"strings"
"github.com/pilosa/pilosa"
)
// GenerateConfigCommand represents a command for printing a default config.
type GenerateConfigCommand struct {
*pilosa.CmdIO
}
// NewGenerateConfigCommand returns a new instance of GenerateConfigCommand.
func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerateConfigCommand {
return &GenerateConfigCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run prints out the default config.
func (cmd *GenerateConfigCommand) Run(ctx context.Context) error {
fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
data-dir = "~/.pilosa"
bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
poll-interval = "2m0s"
replicas = 1
hosts = [
"localhost:10101",
]
[anti-entropy]
interval = "10m0s"
[profile]
cpu = ""
cpu-time = "30s"
[plugins]
path = ""
`)+"\n")
return nil
}

42
ctl/server.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"time"
"github.com/pilosa/pilosa/server"
"github.com/spf13/cobra"
)
// BuildServerFlags attaches a set of flags to the command for a server instance.
func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.")
flags.StringSliceVarP(&srv.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this?
flags.StringVarP(&srv.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.")
flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path")
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.")
flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.")
flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.")
flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]")
flags.StringVarP(&srv.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.")
}

99
docs/administration.md Normal file
View file

@ -0,0 +1,99 @@
+++
title = "Administration Guide"
+++
## Administration Guide
### Installing in production
#### Hardware
Pilosa is a standalone, compiled Go application, so there is no need to worry about running and configuring a Java VM. Pilosa can run on very small machines and works well with even a medium sized dataset on a personal laptop. If you are reading this section, you are likely ready to deploy a cluster of Pilosa servers handling very large datasets or high velocity data. These are guidelines for running a cluster; specific needs may differ.
#### Memory
Pilosa holds all row/column bitmap data in main memory. While this data is compressed more than a typical database, available memory is a primary concern. In a production environment, we recommend choosing hardware with a large amount of memory >= 64GB. Prefer a small number of hosts with lots of memory per host over a larger number with less memory each. Larger clusters tend to be less efficient overall due to increased inter-node communication.
#### CPUs
Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the slice, so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload.
#### Disk
Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs--especially if you have a write heavy application.
#### Network
Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an Indexing service there already should be a system of record, or ability to rebuild a Cluster quickly from backups.
#### Overview
While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
### Importing and Exporting Data
#### Importing
The import API expects a csv of RowID,ColumnID's.
When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use `pilosa sort CSV_FILE` to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
```
pilosa import -d project -f stargazer project-stargazer.csv
```
#### Exporting
Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID.
```
curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \
--header "Accept: text/csv"
```
### Versioning
Pilosa follows [Semantic Versioning](http://semver.org/).
MAJOR.MINOR.PATCH:
* MAJOR version when you make incompatible API changes,
* MINOR version when you add functionality in a backwards-compatible manner, and
* PATCH version when you make backwards-compatible bug fixes.
#### PQL versioning
The Pilosa server should support PQL versioning using HTTP headers. On each request, the client should send a Content-Type header and an Accept header. The server should respond with a Content-Type header that matches the client Accept header. The server should also optionally respond with a Warning header if a PQL version is in a deprecation period, or an HTTP 400 error if a PQL version is no longer supported.
#### Upgrading
When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes.
### Backup/restore
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster.
Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node.
For larger datasets and to make this process faster you could copy the relevant data files from the other nodes to the new one before startup.
Note: This will only work when the replication factor is >= 2
#### Using Index Sync
- Shutdown the cluster.
- Modify config file to replace existing node address with new node.
- Restart all nodes in the cluster.
- Wait for auto Index sync to replicate data from existing nodes to new node.
#### Copying data files manually
- To accomplish this goal you will 1st need:
- List of all Indexes on your cluster
- List of all frames in your Indexes
- Max slice per Index, listed in the /status endpoint
- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
- Using the list of slices owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Frame
- copy each owned slice for an existing node to this new node
- Modify the cluster config file to replace the previous node address with the new node address.
- Restart the cluster
- Wait for the 1st sync (10 minutes) to validate Index connections

245
docs/api-reference.md Normal file
View file

@ -0,0 +1,245 @@
+++
title = "API Reference"
+++
## API Reference
### `/index`
#### `GET`
Returns the schema of all indexes in JSON.
Request:
```
curl -XGET localhost:10101/index
```
Response:
```
{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
```
### `/index/<index-name>`
#### `GET`
Returns the schema of the specified index in JSON.
Request:
```
curl -XGET localhost:10101/index/user
```
Response:
```
{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
```
#### `POST`
Creates an index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `columnLabel` (string): column label of the index.
Request:
```
curl localhost:10101/index/user \
-X POST \
-d '{"options": {"columnLabel": "user_id"}}'
```
Response:
```
{}
```
#### `DELETE`
Removes the given index.
Request:
```
curl -XDELETE localhost:10101/index/user
```
Response:
```
{}
```
### `/index/<index-name>/query`
#### `POST`
Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
Request:
```
curl localhost:10101/index/user/query \
-X POST \
-d 'Bitmap(frame="language", id=5)'
```
Response:
```
{"results":[{"attrs":{},"bits":[100]}]}
```
In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`.
Request:
```
curl localhost:10101/index/user/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="language", id=5)'
```
Response:
```
{
"results":[{"attrs":{},"bits":[100]}],
"columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}]
}
```
### `/index/<index-name>/time-quantum`
#### `PATCH`
Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database.
The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are:
* (Empty string)
* Y: year
* M: month
* D: day
* H: hour
* YM: year and month
* MD: month and day
* DH: day and hour
* YMD: year, month and day
* MDH: month, day and hour
* YMDH: year, month, day and hour
Request:
```
curl localhost:10101/index/user/time-quantum \
-X POST \
-d '{"timeQuantum": "YM"}'
```
Response:
```
{}
```
### `/index/<index-name>/frame/<frame-name>`
#### `POST`
Creates a frame in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields:
* `rowLabel` (string): Row label of the frame.
* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame.
* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`.
* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
Request:
```
curl localhost:10101/index/user/frame/language \
-X POST \
-d '{"options": {"rowLabel": "language_id"}}'
```
Response:
```
{}
```
#### `DELETE`
Removes the given frame.
Request:
```
curl -XDELETE localhost:10101/index/user/frame/language
```
Response:
```
{}
```
### `/index/<index-name>/frame/<frame-name>/time-quantum`
#### `PATCH`
Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame.
The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are:
* (Empty string)
* Y: year
* M: month
* D: day
* H: hour
* YM: year and month
* MD: month and day
* DH: day and hour
* YMD: year, month and day
* MDH: month, day and hour
* YMDH: year, month, day and hour
Request:
```
curl localhost:10101/index/user/frame/language/time-quantum \
-X POST \
-d '{"timeQuantum": "YM"}'
```
Response:
```
{}
```
### `/hosts`
#### `GET`
Returns the hosts in the cluster.
Request:
```
curl -XGET localhost:10101/hosts
```
Response:
```
[{"host":":10101","internalHost":""}]
```
### `/version`
#### `GET`
Returns the version of the Pilosa server.
Request:
```
curl -XGET localhost:10101/version
```
Response:
```
{"version":"v0.3.0-353-ge633247"}
```

226
docs/client-libraries.md Normal file
View file

@ -0,0 +1,226 @@
+++
title = "Client Libraries"
+++
## Client Libraries
### Go
You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-client-pilosa). Check out its [README](https://github.com/pilosa/go-client-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```go
package startrace
import (
"fmt"
pilosa "github.com/pilosa/go-client-pilosa"
)
func main() {
// Let's create Index and Frame objects, which will contain the settings
// for the corresponding indexes and frames.
repositoryOptions, := &pilosa.ColumnOptions{ColumnLabel: "repo_id"}
repository, _ := pilosa.NewIndex("repository", repositoryOptions)
stargazerOptions := &pilosa.RowOptions{RowLabel: "stargazer_id"}
stargazer, _ := repository.Frame("stargazer", stargazerOptions)
languageOptions := &pilosa.RowOptions{RowLabel: "language_id"}
language, _ := repository.Frame("language", languageOptions)
// We will just use the default client which assumes the server is at http://localhost:10101
client := pilosa.DefaultClient()
var response *pilosa.QueryResponse
var result *pilosa.QueryResult
// Which repositories did user 8 star:
response, _ = client.Query(stargazer.Bitmap(8), nil)
result = response.Result()
if result != nil {
fmt.Println("User 8 starred: ", result.Bitmap.Bits)
}
// What are the top 5 languages in the sample data:
response, _ = client.Query(language.TopN(5), nil)
if result != nil {
fmt.Println("Top 5 languages: ", result.Bitmap.Bits)
}
// Which repositories were starred by user 8 and 18:
response, _ = client.Query(
repository.Intersect(
stargazer.Bitmap(8),
stargazer.Bitmap(18)),
nil)
result = response.Result()
if result != nil {
fmt.Println("Repositories starred by both user 8 and 18: ", result.Bitmap.Bits)
}
// Which repositories were starred by user 8 and 18 and also were written in language 1
response, _ = client.Query(
repository.Intersect(
stargazer.Bitmap(8),
stargazer.Bitmap(18),
language.Bitmap(1)),
nil)
result = response.Result()
if result != nil {
fmt.Println("Repositories starred by both user 8 and 18 and are in language 1: ", result.Bitmap.Bits)
}
// Set user 99999 as a stargazer for repository 77777:
_, err = client.Query(stargazer.SetBit(99999, 77777), nil)
if err != nil {
fmt.Println("Error setting bit: ", err)
}
}
```
### Python
You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.rst) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```python
from pilosa import Index, Client, PilosaError
# Let's create Index and Frame objects, which will contain the settings
# for the corresponding indexes and frames.
repository = Index("repository", column_label="repo_id")
stargazer = repository.frame("stargazer", row_label="stargazer_id")
language = repository.frame("language", row_label="language_id")
# We will just use the default client which assumes the server is at http://localhost:10101
client = Client()
# Which repositories did user 8 star:
response = client.query(stargazer.bitmap(8))
if response.result:
print("User 8 starred: ", result.bitmap.bits)
# What are the top 5 languages in the sample data:
response = client.query(language.topn(5))
if response.result:
print("Top 5 languages: ", result.bitmap.bits)
# Which repositories were starred by user 8 and 18:
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18)))
if response.result:
print("Repositories starred by both user 8 and 18: ", result.bitmap.bits)
# Which repositories were starred by user 8 and 18 and also were written in language 1
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18),
language.bitmap(1)))
if response.result:
print("Repositories starred by both user 8 and 18 and are in language 1: ", result.bitmap.bits)
# Set user 99999 as a stargazer for repository 77777
try:
client.query(stargazer.setbit(99999, 77777))
except PilosaError as ex:
print("Error setting bit: ", ex)
```
### Java
You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions.
We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`.
Error handling has been omitted in the example below for brevity.
```java
import com.pilosa.client.*;
import com.pilosa.client.orm.*;
public class StarTrace {
public static void main(String[] args) {
// Let's create Index and Frame objects, which will contain the settings
// for the corresponding indexes and frames.
IndexOptions repositoryOptions = IndexOptions.builder()
.setColumnLabel("repo_id")
.build();
Index repository = Index.withName("repository", repositoryOptions);
FrameOptions stargazerOptions = FrameOptions.builder()
.setRowLabel("stargazer_id")
.build();
Frame stargazer = repository.frame("stargazer", stargazerOptions);
FrameOptions languageOptions = FrameOptions.builder()
.setRowLabel("language_id")
.build();
Frame language = repository.frame("language", languageOptions);
// We will just use the default client which assumes the server is at http://localhost:10101
PilosaClient client = PilosaClient.defaultClient();
QueryResponse response;
QueryResult result;
// Which repositories did user 8 star:
response = client.query(stargazer.bitmap(8));
result = response.getResult();
if (result != null) {
System.out.println("User 8 starred: " + result.getBitmap().getBits());
}
// What are the top 5 languages in the sample data:
response = client.query(language.topN(5));
result = response.getResult();
if (result != null) {
System.out.println("Top 5 languages: " + result.getBitmap().getBits());
}
// Which repositories were starred by user 8 and 18:
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18)));
result = response.getResult();
if (result != null) {
System.out.println("Repositories starred by both user 8 and 18: "
+ result.getBitmap().getBits());
}
// Which repositories were starred by user 8 and 18 and also were written in language 1
response = client.query(
repository.intersect(
stargazer.bitmap(8),
stargazer.bitmap(18),
language.bitmap(1)));
result = response.getResult();
if (result != null) {
System.out.println("Repositories starred by both user 8 and 18 and are in language 1: "
+ result.getBitmap().getBits());
}
// Set user 99999 as a stargazer for repository 77777:
try {
client.query(stargazer.setBit(99999, 77777))
}
catch (PilosaException ex) {
System.out.println("Error setting bit: " + ex)
}
}
}
```

169
docs/configuration.md Normal file
View file

@ -0,0 +1,169 @@
+++
title = "Configuration"
+++
## Configuration
Pilosa can be configured through command line flags, environment variables, and/or a configuration file; configured options take precedence in that order. So if an option is specified in a command line flag, it will take precedence over the same option specified in the environment, which would take precedence over that same option specified in the configuration file.
All options are available in all three configuration types with the exception of the `--config` option which specifies the location of the config file, and therefore will not be used if it is present in the config file.
The syntax for each option is slightly different between each of the configuration types, but follows a simple formula. See the following three sections for an explanation of each configuration type.
### Command line flags
Pilosa uses GNU/POSIX style flags. Most flags you specify as `--flagname=value` although some have a short form that is a single character and can be specified with a single dash like `-f value`. Running `pilosa server --help` will give an overview of the available flags as well as their short forms (if applicable).
### Environment variables
Every command line flag has a corresponding environment variable. The environment variable is the flag name in all caps, prefxed by `PILOSA_`, and with any dashes replaced by underscores. For example: `--flag-name` becomes `PILOSA_FLAG_NAME`.
### Config file
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file:
```toml
[cluster]
poll-interval = "2m0s"
replicas = 1
```
Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes:
```toml
[cluster]
hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"]
```
### All Options
#### Anti Entropy Interval
* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync.
* Flag: `--anti-entropy.interval="10m0s"`
* Env: `PILOSA_ANTI_ENTROPY.INTERVAL="10m0s"`
* Config:
```toml
[anti-entropy]
interval = "10m0s"
```
#### Bind
* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101.
* Flag: `--bind="localhost:10101"`
* Env: `PILOSA_BIND="localhost:10101"`
* Config:
```toml
bind = localhost:10101
```
#### Cluster Hosts
* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms.
* Flag: `--cluster.hosts="localhost:10101"`
* Env: `PILOSA_CLUSTER.HOSTS="localhost:10101"`
* Config:
```toml
[cluster]
hosts = ["localhost:10101"]
```
#### Cluster Internal Hosts
* Description: List of hosts in the cluster used for internal communication. Multiple hosts should be comma separated in the flag and env forms.
* Flag: `--cluster.internal-hosts="localhost:11101"`
* Env: `PILOSA_CLUSTER.INTERNAL_HOSTS="localhost:11101"`
* Config:
```toml
[cluster]
internal-hosts = ["localhost:11101"]
```
#### Cluster Internal Port
* Description: Port to which Pilosa should bind for internal communication.
* Flag: `--cluster.internal-port=11101`
* Env: `PILOSA_CLUSTER.INTERNAL_PORT=11101`
* Config:
```toml
[cluster]
internal-port = 11101
```
#### Cluster Poll Interval
* Description: Polling interval for cluster.
* Flag: `cluster.poll-interval="1m0s"`
* Env: `PILOSA_CLUSTER.POLL_INTERVAL="1m0s"`
* Config:
```toml
[cluster]
poll-interval = "1m0s"
```
#### Cluster Replicas
* Description: Number of hosts each piece of data should be stored on.
* Flag: `cluster.replicas=1`
* Env: `PILOSA_CLUSTER.REPLICAS=1`
* Config:
```toml
[cluster]
replicas = 1
```
#### Cluster Type
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip].
* static - Messaging between nodes is disabled. This is primarily used for testing.
* http - Messages are transmitted over HTTP.
* gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip.
* Flag: `cluster.type="gossip"`
* Env: `PILOSA_CLUSTER.TYPE="gossip"`
* Config:
```toml
[cluster]
type = "gossip"
```
#### Data Dir
* Description: Directory to store Pilosa data files.
* Flag: `--data-dir="~/.pilosa"`
* Env: `PILOSA_DATA_DIR="~/.pilosa"`
* Config:
```toml
data-dir = "~/.pilosa"
```
#### Profile CPU
* Description: If this is set to a path, collect a cpu profile and store it there.
* Flag: `--profile.cpu="/path/to/somewhere"`
* Env: `PILOSA_PROFILE.CPU="/path/to/somewhere"`
* Config:
```toml
[profile]
cpu = "/path/to/somewhere"
```
#### Profile CPU Time
* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set.
* Flag: `--profile.cpu-time="30s"`
* Env: `PILOSA_PROFILE.CPU_TIME="30s"
* Config:
```toml
[profile]
cpu-time = "30s"
```

91
docs/data-model.md Normal file
View file

@ -0,0 +1,91 @@
+++
title = "Data Model"
+++
## Data Model
### Overview
The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit - if the bit is set, it indicates that a relationship exists between that particular row and column.
Rows and columns can represent anything (they could even represent the same set of things). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row.
![data model diagram](/img/docs/data-model.svg)
### Index
The purpose of the Index is to represent a data namespace. You cannot perform cross-index queries. Column-level attributes are global to the Index.
### Column
Column ids are sequential increasing integers and are common to all Frames within an Index.
### Row
Row ids are sequential increasing integers namespaced to each Frame within an Index.
### Frame
Frames are used to segment and define different functional characteristics within your entire index. You can think of a Frame as a table-like data partition within your Index.
Row attributes are namespaced at the Frame level.
#### Ranked
Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation.
![ranked frame diagram](/img/docs/frame-ranked.svg)
#### LRU
The LRU cache maintains the most recently accessed Rows.
![lru frame diagram](/img/docs/frame-lru.svg)
### Time Quantum
Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
### Attribute
Attributes are arbitrary key/value pairs that can be associated to both rows or columns. This metadata is stored in a separate BoltDB data structure.
### Slice
Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth.
Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
### View
Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation.
#### Standard
The standard View contains the same Row/Column format as the input data.
#### Inverse
The Inverse View contains the same data with the Row and Column swapped.
For example, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3)
SetBit(frame="A", rowID=11, columnID=3)
SetBit(frame="A", rowID=19, columnID=5)
```
![inverse frame diagram](/img/docs/frame-inverse.svg)
#### Time Quantums
If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below:
```
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00")
SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00")
```
![time quantum frame diagram](/img/docs/frame-time-quantum.svg)

40
docs/faq.md Normal file
View file

@ -0,0 +1,40 @@
+++
title = "FAQ"
+++
## FAQ
### What is Pilosa?
Pilosa is an in-memory, distributed index that is layered over persistent storage. It supports fast ad-hoc queries and segmentation. Pilosa does not require the underlying data to be moved, rather it can be populated in conjunction with data writes, or it can be backfilled asynchronously from any other data store or event processing system. This allows Pilosa to support sub-second queries against very large underlying data sets.
### Is Pilosa a database?
Pilosa is not a database in the traditional sense. While Pilosa does store data (both in-memory as well as persisted to disk), it wouldn't typically be used as a primary data store. Instead, one would likely use Pilosa as an index of the data stored in a traditional database or in a data warehouse.
### Where does Pilosa fit in my stack?
Pilosa sits on top of a data store or multiple data stores.
How is Pilosa different than Elasticsearch since they are both indexes?
Elasticsearch is a search engine based on Lucene, and is therefore very good at indexing and searching large volumes of unstructured text. As it matures, Elasticsearch has continued to move into the analytics space, but its core data object is still the "document". Pilosa is specifically designed to index structured data and improve query speed. By representing data as the relationship between objects, and then storing those relationships in bitmaps, Pilosa can very efficiently search and compare many millions of data points while still maintaining a small memory footprint.
### How do I get my data into Pilosa?
There are typically two methods for getting data into Pilosa: importing large batches of data from an existing data set, and continuously updating Pilosa as data is added or updated.
In the first case, one would use the `pilosa import` command to bulk load structured data into Pilosa. In order to improve this process, one can use the Pilosa Development Kit (PDK) to map structured data in the original data set onto the Pilosa schema.
For the case where data is continually mutating, one would apply a parallel data writer at the point at which data is written to the persistent data store. This new writer would simultaneously write to Pilosa. An example use case would be one where Kafka was employed as the message broker in your data pipeline, you could introduce an additional Kafka consumer to read from the message log and write mutated data to Pilosa.
### What languages can I use with it?
There is currently client support for Go, Python, and Java. If you want to use Pilosa with a different language, you can access Pilosa via the Pilosa API.
### Do you query Pilosa using SQL?
One can access Pilosa directly via the terminal using the Pilosa Query Language (PQL), but a typical implementation would use one of the Pilosa client libraries to integrate with an existing codebase. There is currently client support for Go, Python, and Java.
### Replication on each node?
Pilosa supports a replication factor greater than or equal to one. When replication is configured to be greater than one, then all mutations will be replicated to additional nodes in the cluster. For example, in a five-node cluster consisting of nodes A-B-C-D-E and with replication factor of three, then a write to node B will result in data being written to nodes B, C, and D. If the replication factor is greater than the number of nodes in the cluster, the data will be replicated to every node in the cluster only once.

153
docs/getting-started.md Normal file
View file

@ -0,0 +1,153 @@
+++
title = "Getting Started"
+++
## Getting Started
Pilosa supports an HTTP interface which uses JSON by default.
Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html).
<div class="note">
<p>Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit.</p>
</div>
### Starting Pilosa
Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa.
Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`):
```
pilosa server
```
If you are using the Docker image, you can run an ephemeral Pilosa container on the default address using the following command:
```
docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest
```
Let's make sure Pilosa is running:
```
curl localhost:10101/status
```
Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}`
### Sample Project
In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation.
#### Create the Schema
Note:
The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows:
```
curl localhost:10101/schema
{"indexes":null}
```
Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first:
```
curl localhost:10101/index/repository \
-X POST \
-d '{"options": {"columnLabel": "repo_id"}}'
```
Repository IDs are the main focus of the `repository` index, so we chose `repo_id` as the column label.
Let's create the `stargazer` frame which has user IDs of stargazers as its rows:
```
curl localhost:10101/index/repository/frame/stargazer \
-X POST \
-d '{"options": {"rowLabel": "stargazer_id",
"timeQuantum": "YMD",
"inverseEnabled": true}}'
```
Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
We set `inverseEnabled` to `true` in order to allow queries over columns as well as rows.
Next up is the `language` frame, which will contain IDs for programming languages:
```
curl localhost:10101/index/repository/frame/language \
-X POST \
-d '{"options": {"rowLabel": "language_id",
"inverseEnabled": true}}'
```
#### Import Some Data
The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download the `stargazer.csv` and `language.csv` files in that repo.
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
```
Run the following commands to import the data into Pilosa:
```
pilosa import -i repository -f stargazer stargazer.csv
pilosa import -i repository -f language language.csv
```
If you are using a Docker container for Pilosa (with name `pilosa`), you should instead copy the `*.csv` file into the container and then import them:
```
docker cp stargazer.csv pilosa:/stargazer.csv
docker exec -it pilosa /pilosa import -i repository -f stargazer /stargazer.csv
docker cp language.csv pilosa:/language.csv
docker exec -it pilosa /pilosa import -i repository -f language /language.csv
```
Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages.
#### Make Some Queries
<div class="note">
<p>Note the Pilosa server comes with a <a href="../webui/">WebUI</a> for constructing queries in a browser. In local development, it is available at <a href="http://localhost:10101">localhost:10101</a>.</p>
</div>
Which repositories did user 14 star:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'Bitmap(frame="stargazer", stargazer_id=14)'
```
What are the top 5 languages in the sample data:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'TopN(frame="language", n=5)'
```
Which repositories were starred by user 14 and 19:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19))'
```
Which repositories were starred by user 14 or 19:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19))'
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19), Bitmap(frame="language", language_id=1))'
```
Set user 99999 as a stargazer for repository 77777:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", repo_id=77777, stargazer_id=99999)'
```
### What's Next?
You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).

56
docs/glossary.md Normal file
View file

@ -0,0 +1,56 @@
+++
title = "Glossary"
+++
## Glossary
<strong id="index">Index:</strong> Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes.
<strong id="column">Column:</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index.
<strong id="row">Row:</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index.
<strong id="bit">Bit:</strong> A bit is the intersection of a Row and Column.
<strong id="bitmap">Bitmap:</strong> The on-disk and in-memory representation of a Row.
<strong id="roaring-bitmap">Roaring Bitmap:</strong> [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses.
<strong id="attribute">Attribute:</strong> Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store.
<strong id="pql">PQL:</strong> Pilosa Query Language
<strong id="index">Index:</strong> The Index represents a data namespace.
<strong id="frame">Frame:</strong> Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame.
<strong id="view">View:</strong> Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a frame and slice in an index.
<strong id="slice">Slice:</strong> Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash.
<strong id="slicewidth">SliceWidth:</strong> This is the default number of columns in a slice.
<strong id="maxslice">MaxSlice:</strong> The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries.
<strong id="anti-entropy">Anti-entropy:</strong> A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies.
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a cluster.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="topn">TopN:</strong> Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row.
<strong id="tanimoto">Tanimoto:</strong> Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity.
<strong id="protobuf">Protobuf:</strong>: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="toml">TOML:</strong> We use [TOML](https://github.com/toml-lang/toml) for our configuration file format.
<strong id="jump-consistent-hash">Jump Consistent Hash:</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
https://arxiv.org/pdf/1406.2294v1.pdf
<strong id="partition">Partition:</strong> The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography.
<strong id="replica">Replica:</strong> A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made).

349
docs/installation.md Normal file
View file

@ -0,0 +1,349 @@
+++
title = "Installation"
+++
## Installation
Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#installing-on-linux).
### Installing on MacOS
There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker.
#### Download the Binary
1. Download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-darwin-amd64.tar.gz
```
Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.3.1-darwin-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.3.1-darwin-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at http://pilosa.com/docs
Version: v0.3.0-279-gcf7082f
Build Time: 2017-04-21T15:36:08+0000
Usage:
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the default configuration.
export Export data from pilosa.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
sort Sort import data for optimal import performance.
Flags:
-c, --config string Configuration file to read from.
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Build from Source
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
* [Glide](http://glide.sh/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
```
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at http://pilosa.com/docs
Version: v0.3.0-279-gcf7082f
Build Time: 2017-04-21T15:36:08+0000
Usage:
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the default configuration.
export Export data from pilosa.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
sort Sort import data for optimal import performance.
Flags:
-c, --config string Configuration file to read from.
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Use Docker
1. Install Docker for Mac.
2. Confirm that the Docker daemon is running in the background:
```
docker version
```
If you don't see the server listed, start the Docker application.
3. Pull the official Pilosa image from Docker Hub:
```
docker pull pilosa/pilosa:latest
```
4. Make sure Pilosa is installed successfully:
```
docker run --rm pilosa/pilosa:latest help
```
#### What's next?
Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
### Installing on Linux
There are three ways to install Pilosa on Linux: download the binary (recommended), build from source, or use Docker.
#### Download the Binary
1. To install the latest version of Pilosa, download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-linux-amd64.tar.gz
```
Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v0.3.1-linux-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v0.3.1-linux-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at http://pilosa.com/docs
Version: v0.3.0-279-gcf7082f
Build Time: 2017-04-21T15:36:08+0000
Usage:
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the default configuration.
export Export data from pilosa.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
sort Sort import data for optimal import performance.
Flags:
-c, --config string Configuration file to read from.
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Build from Source
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
* [Glide](http://glide.sh/)
2. Clone the repo:
```
go get -d github.com/pilosa/pilosa
```
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at http://pilosa.com/docs
Version: v0.3.0-279-gcf7082f
Build Time: 2017-04-21T15:36:08+0000
Usage:
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the default configuration.
export Export data from pilosa.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
sort Sort import data for optimal import performance.
Flags:
-c, --config string Configuration file to read from.
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Use Docker
1. Install Docker.
2. Confirm that the Docker daemon is running in the background:
```
docker version
```
If you don't see the server listed, start the Docker application.
3. Pull the official Pilosa image from Docker Hub:
```
docker pull pilosa/pilosa:latest
```
4. Make sure Pilosa is installed successfully:
```
docker run --rm pilosa/pilosa:latest help
```
#### What's next?
Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
<!--
### Windows
Windows is currently not supported as a target deployment platform for Pilosa, but developing and running Pilosa is made possible by Windows Subsystem for Linux and Docker. See the [Docker](#docker) documentation for using Docker for Windows and Docker Toolbox. You can find documentation about installing Windows Subsystem for Linux at https://msdn.microsoft.com/en-us/commandline/wsl/install_guide. From there, use the instructions in the [Linux Install](#installing-on-linux) section in the this document.
### Docker
1. Install Docker for your platform. On Linux, Docker is available via your package manager. On MacOS, you can use Docker for Mac or Docker Toolbox. On Windows, you can use Docker for Windows or Docker Toolbox.
2. **This step is necessary only if you are using Docker Toolbox**:
a. Start the Docker support using `docker-machine start` in a terminal. The environment variables of the terminal should be updated accordingly, run `docker-machine env` to display the necessary commands.
b. Set up port forwarding in the VirtualBox GUI or on the command line. Guest port should be 10101. For the host port, 10101 is recommended. If the `VBoxManage` command is in your `PATH`, you can use the following command (assuming you use the default VM):
```
VBoxManage modifyvm "default" --natpf1 "pilosa,tcp,,10101,,10101"
```
3. Confirm that the Docker daemon is running in the background:
```
docker version
```
If you don't see the server listed, start the Docker application.
4. Pull the official Pilosa image from Docker Hub:
```
docker pull pilosa/pilosa:latest
```
5. Make sure Pilosa is installed successfully:
```
docker run --rm pilosa/pilosa:latest version
```
-->

17
docs/introduction.md Normal file
View file

@ -0,0 +1,17 @@
+++
title = "Introduction"
+++
## Introduction
Pilosa is an open source, distributed bitmap index.
[//]: # (TODO insert a graphic here?)
It is designed primarly for speed and horizontal scalability. If you have data with billions of objects that can have millions of possible attributes, and you want to explore those relationships, Pilosa can help you.
"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.

62
docs/pdk.md Normal file
View file

@ -0,0 +1,62 @@
+++
title = "PDK"
+++
## PDK
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly.
The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary.
### Library
#### Mapping
Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts:
* an enumeration of field names
* a list of parsers that are used to parse strings in the CSV to values
* a list of commonly used, named, mapper functions
* a list of ParserMappers - objects that encapsulate all of the work related to a single frame.
This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK.
With this definition available, the PDK tool can run the import, which consists of these steps:
- create the index
- create all frames
- for each CSV file, read all rows
- for each CSV record:
- generate a columnID
- apply all ParserMappers, generating a list of (frame, ID) pairs
- set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID)
The process is summarized in this flowchart:
![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg)
Some of the simple mapper functions available with PDK include:
* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year.
* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11].
* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6].
* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23].
* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins.
* BoolMapper: Maps a boolean value to the range [0, 1].
* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain.
* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small.
* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets.
* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket.
* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example.
* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper.
### Examples
Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help <example>` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality.
<!--
#### Net
A detailed discussion of using Pilosa to index network traffic data is available [here - TODO]link blog post). This will discuss the implementation of `pdk net` as it relates to the use of the PDK library tools.
-->

395
docs/query-language.md Normal file
View file

@ -0,0 +1,395 @@
+++
title = "Query Language"
+++
## Query Language
This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
```
{"results":[...]}
```
There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type.
Row and Column labels are set and frame and index creation time respectively. When the specification of a query says *row_label* or *col_label*, one should use the labels that were set while creating the index and frame. The default row label is `id`, and the default column label is `columnID`.
#### Conventions
* Angle Brackets `<>` denote required arguments
* Square Brackets `[]` denote optional arguments
* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`)
##### Examples
Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data.
The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", repo_id=10, stargazer_id=1)` against a server using curl, you would:
```
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", repo_id=10, stargazer_id=1)'
```
#### Arguments and Types
* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length.
* `ROW_LABEL` Pilosa allows users to set different row labels for each frame at frame creation time. The default row label is `rowID`, but one may set a more descriptive row label for their data (such as `stargazer_id`).
* `COL_LABEL` Pilosa allows users to set a different column label for each index at index creation time. The default column label is `columnID`.
* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04")
* `UINT` An unsigned integer (e.g. 42839)
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
* `ATTR_VALUE` Can be a string, float, integer, or bool.
* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range`
* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
#### Write Operations
##### SetBit
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
[timestamp=TIMESTAMP])
```
**Description:**
`SetBit`, assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column.
**Result Type:** boolean
A return value of `true` indicates that the bit was changed to 1.
A return value of `false` indicates that the bit was already set to 1 and nothing changed.
**Examples:**
```
SetBit(frame="stargazer", repo_id=10, stargazer_id=1)
```
This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10.
SetBit also supports providing a timestamp. To write the date that a user starred a repository.
```
SetBit(frame="stargazer", repo_id=10, stargazer_id=1, timestamp="2016-01-01T00:00")
```
Setting multiple bits in a single request:
```
SetBit(frame="stargazer", repo_id=10, stargazer_id=1) SetBit(frame="stargazer", repo_id=10, stargazer_id=2) SetBit(frame="stargazer", repo_id=20, stargazer_id=1) SetBit(frame="stargazer", repo_id=30, stargazer_id=2)
```
##### SetRowAttrs
**Spec:**
```
SetRowAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
**Description:**
`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute.
**Result Type:** null
SetRowAttrs queries always return `null` upon success.
**Examples:**
```
SetRowAttrs(frame="stargazer", stargazer_id=10, username="mrpi", active=true)
```
Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`.
```
SetRowAttrs(frame="stargazer", stargazer_id=10, username=null)
```
Delete username value for user 10.
##### SetColumnAttrs
**Spec:**
```
SetColumnAttrs(<frame=STRING>, <ROW_LABEL=UINT>,
<ATTR_NAME=ATTR_VALUE>,
[ATTR_NAME=ATTR_VALUE ...])
```
**Description:**
`SetColumnAttrs` associates arbitrary key/value pairs with a column in an index.
**Result Type:** null
SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute.
**Examples:**
```
SetColumnAttrs(frame="stargazer", repo_id=10, stars=123, url="http://projects.pilosa.com/10", active=true)
```
Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`.
```
SetColumnAttrs(frame="stargazer", repo_id=10, url=null)
```
Delete url value for repo 10.
##### ClearBit
**Spec:**
```
SetBit(<frame=STRING>, <ROW_LABEL=UINT>, <COL_LABEL=UINT>,
[timestamp=TIMESTAMP])
```
**Description:**
`ClearBit`, assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column.
**Result Type:** boolean
A return value of `true` indicates that the bit was toggled from 1 to 0.
A return value of `false` indicates that the bit was already set to 0 and nothing changed.
**Examples:**
```
ClearBit(frame="stargazer", repo_id=10, stargazer_id=1)
```
Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame.
#### Read Operations
##### Bitmap
**Spec:**
```
Bitmap(<frame=STRING>, (<ROW_LABEL=UINT> | <COL_LABEL>=UINT))
```
**Description:**
`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column.
**Result Type:** object with attrs and bits.
e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
**Examples:**
Query all repositories that user 1 has starred.
```
Bitmap(frame="stargazer", stargazer_id=1)
```
Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
* attrs are the attributes for user 1
* bits are the repositories which user 1 has starred.
##### Union
**Spec:**
```
Union([BITMAP_CALL ...])
```
**Description:**
Union performs a logical OR on the results of each `BITMAP_CALL` query passed to it.
**Result Type:** object with attrs and bits
attrs will always be empty
**Examples:**
Query all repositories that are contributed by multiple users
```
Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2))
```
Returns `{"attrs":{},"bits":[10, 20, 30]}`.
* bits are repositories that were starred by user 1 OR user 2
##### Intersect
**Spec:**
```
Intersect(<BITMAP_CALL>, [BITMAP_CALL ...])
```
**Description:**
Intersect performs a logical AND on the results of each `BITMAP_CALL` query passed to it.
**Result Type:** object with attrs and bits
attrs will always be empty
**Examples:**
Query repositories which have been starred by two users.
```
Intersect(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2))
```
Returns `{"attrs":{},"bits":[10]}`.
* bits are repositories that were starred by user 1 AND user 2
##### Difference
**Spec:**
```
Difference(<BITMAP_CALL>, [BITMAP_CALL ...])
```
**Description:**
Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`.
**Result Type:** object with attrs and bits
attrs will always be empty
**Examples:**
Query repositories which have been starred by one user and not another.
```
Difference(Bitmap(frame="stargazer", stargazer_id=1), Bitmap( frame="stargazer", stargazer_id=2))
```
Return `{"results":[{"attrs":{},"bits":[20]}]}`
* bits are repositories that were starred by user 1 BUT NOT user 2
```
Difference(Bitmap(frame="stargazer", stargazer_id=2), Bitmap( frame="stargazer", stargazer_id=1))
```
Return `{"attrs":{},"bits":[30]}`
* Bits are repositories that were starred by user 2 BUT NOT user 1
##### Count
**Spec:**
```
Count(<BITMAP_CALL>)
```
**Description:**
Returns the number of set bits in the `BITMAP_CALL` passed in.
**Result Type:** int
**Examples:**
Query the number of repositories to which a user has contributed.
```
Count(Bitmap(frame="stargazer", stargazer_id=1))
```
Return `2`
* Result is the number of repositories that user 1 has starred.
##### TopN
**Spec:**
```
TopN([BITMAP_CALL], <frame=STRING>, [n=UINT],
[<field=ATTR_NAME>, <filters=[]ATTR_VALUE>])
```
**Description:**
Return the id and count of the top `n` bitmaps (by count of bits) in the frame.
The `field` and `filters` arguments work together to only return Bitmaps which
have the attribute specified by `field` with one of the values specified in
`filters`.
**Result Type:** array of key/count objects
**Examples:**
```
TopN(frame="stargazer")
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]`
* key is a user
* count is amount of repositories
* Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository.
```
TopN(frame="stargazer", n=2)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]`
* Results are the top two users sorted by number of repositories they've starred in descending order.
```
TopN(Bitmap(frame="language", language_id=1), frame="stargazer", n=2)
```
Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
* Results are the top two users sorted by the number of repositories that they've starred which are written in language 1.
##### Range Queries
**Spec:**
```
Range(<frame=STRING>, <ROW_LABEL=UINT>,
<start=TIMESTAMP>, <end=TIMESTAMP>)
```
**Description:**
Similar to `Bitmap`, but only returns bits which were set with timestamps
between the given `start` and `end` timestamps.
**Result Type:** object with attrs and bits
**Examples:**
When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range.
```
Range(frame="stargazer", stargazer_id=1, start="2010-01-01T00:00", end="2017-03-02T03:00")
```
Returns `{{"attrs":{},"bits":[10]}`
* bits are repositories which were starred by user 1 from 2010-01-01 to 2017-03-02

342
docs/tutorials.md Normal file
View file

@ -0,0 +1,342 @@
+++
title = "Tutorials"
+++
## Tutorials
### Transportation
#### Introduction
New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set.
Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
#### Data Model
The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at:
* Distance: miles, floating point
* Fare: dollars, floating point
* Number of passengers: integer
* Dropoff location: latitude and longitude, floating point
* Pickup location: latitude and longitude, floating point
* Dropoff time: timestamp
* Pickup time: timestamp
We import these fields, creating one or more Pilosa frames from each of them:
frame |mapping
------------|---------------------
cab_type |direct map of enum int → row ID
dist_miles |round(dist) → row ID
total_amount_dollars |round(dist) → row ID
passenger_count |direct map of integer value → row ID
drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID
drop_year |year(timestamp) → row ID
drop_month |month(timestamp) → row ID
drop_day |day(timestamp) → row ID
drop_time |time of day mapped to one of 48 half-hour buckets
pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID
pickup_year |year(timestamp) → row ID
pickup_month |month(timestamp) → row ID
pickup_day |day(timestamp) → row ID
pickup_time |time of day mapped to one of 48 half-hour buckets → row ID
We also created two extra frames that represent the duration and average speed of each ride:
frame |mapping
--------------------|-------------
duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID
speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
#### Mapping
Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
##### 0 columns → 1 frame
cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant.
##### 1 column → 1 frame
The following three frames are mapped in a simple direct way from single columns of the original data.
dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code:
```go
lfm := pdk.LinearFloatMapper{
Min: -0.5,
Max: 3600.5,
Res: 3601,
}
```
`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three.
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`).
```go
pdk.BitMapper{
Frame: "dist_miles",
Mapper: lfm,
Parsers: []pdk.Parser{pdk.FloatParser{}},
Fields: []int{fields["trip_distance"]},
},
```
These same objects are represented in the JSON definition file:
```go
{
"Fields": [
"Trip_distance": 10,
]
"Mappers": [
{
"Name": "lfm0",
"Min": -0.5,
"Max": 3600.5,
"Res": 3600
},
],
"BitMappers": [
{
"Frame": "dist_miles",
"Mapper": {
"Name": "lfm0"
},
"Parsers": [
{"Name": "FloatParser"}
],
"Fields": "Trip_distance",
}
]
}
```
Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable.
**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one.
**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
##### 1 column → multiple frames
When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day".
We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day".
We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
##### Multiple columns → 1 frame
The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id.
Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
##### Complex mappings
We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
```go
durm := pdk.CustomMapper{
Func: func(fields ...interface{}) interface{} {
start := fields[0].(time.Time)
end := fields[1].(time.Time)
return end.Sub(start).Minutes()
},
Mapper: lfm,
}
```
#### Import process
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process.
#### Queries
Now we can run some example queries.
Count per cab type can be retrieved, sorted, with a single PQL call.
```
TopN(frame=cab_type)
```
High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
```
TopN(frame=pickup_grid_id)
```
Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
```python
queries = ''
pcounts = range(10)
for i in pcounts:
queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i
resp = requests.post(qurl, data=queries)
average_amounts = []
for pcount, topn in zip(pcounts, resp.json()['results']):
wsum = sum([r['count'] * r['key'] for r in topn])
count = sum([r['count'] for r in topn])
average_amounts.append(float(wsum)/count)
```
For more examples and details, see this [ipython notebook](https://github.com/alanbernstein/pilosa-notebooks/blob/master/taxi-use-case.ipynb).
### Chemical similarity search
#### Overview
The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals.
We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set.
Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient.
```
T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B))
```
A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical.
All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase
#### Data model
We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id.
Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions.
Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with:
```
SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23"
threshold = 90
```
return the set of molecules that have at least a 90% similarity with the given molecule.
The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint.
Standard View is used to calculate similarity
```
Index: mole
View: Standard
Col: chembl_id
Frame: fingerprint
Row: position_id ("on" bit positions of a fingerprint)
```
Inverse View is used for finding chembl_id based on given SMILES.
From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint.
```
Index: mole
View: Inverse
Col: position_id ("on" bit positions of a fingerprint)
Frame: fingerprint
Row: chembl_id
```
After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold.
#### Import process
To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key.
For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase:
```
python import_from_sdf.py -p <path_to_sdf_file> -file id_fingerprint.csv
```
First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above.
The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393
```
curl localhost:10101/index/mole \
-X POST \
-d '{"options": {"columnLabel": "position_id"}}'
curl localhost:10101/index/mole/frame/fingerprint \
-X POST \
-d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}'
```
Run the following commands to import the csv data into the `mole` index:
```
pilosa import -d mole -f fingerprint id_fingerprint.csv
```
#### Queries
Get chembl_id from a given SMILES:
```
python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23"
```
Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps:
* Convert SMILES to fingerprint bit "on" positions
```python
from rdkit import Chem
from rdkit.Chem import AllChem
mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23")
fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits())
```
* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id
```python
bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp]
bitmap_string = ', '.join(bit_maps)
intersection = "Intersect(%s)" % bitmap_string
mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"]
```
* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist.
```python
for m in mole_ids:
mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"]
existed_mol = False
if len(mol) == len(fp):
found = m
existed_mol = True
break
```
Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70%
```
python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70
```
Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps:
* Get chembl_id from a SMILES (steps discussed above)
* Query Pilosa’s TopN to get list of similarity chembl_id
```python
query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)'
topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string)
```
#### Benchmark
To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command:
```
python benchmarks.py -id 6223
```
As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa.
Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster

36
docs/webui.md Normal file
View file

@ -0,0 +1,36 @@
+++
title = "WebUI"
+++
## WebUI
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101)
This can be used for constructing queries and viewing the cluster status.
### Console
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
The Console will keep a record of each query and its result with the latest query on top.
![console](/img/docs/webui-console.png)
In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
- `:create index <indexname>`
- `:delete index <indexname>`
- `:use <indexname>`
- `:create frame <framename>`
- `:delete frame <framename>`
Index and frame creation also supports options like `columnLabel`,`rowLabel` or `inverseEnabled`. When creating new index or new frame, add options by using the keys documented in [API reference](../api-reference).
- `:create index <indexname> columnLabel=col_id`
- `:create frame <framename> rowLabel=row_id inverseEnabled=true cacheSize=10000`
### Cluster Admin
Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.

2
glide.lock generated
View file

@ -67,7 +67,7 @@ imports:
- name: github.com/pelletier/go-buffruneio
version: c37440a7cf42ac63b919c752ca73a85067e05992
- name: github.com/pelletier/go-toml
version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a
version: 23f644976aa7c724adf4aec911dadf4af17840ab
- name: github.com/rakyll/statik
version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f
subpackages:

1
handler.go
View file

@ -291,6 +291,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) {
} else if _, err := w.Write(buf); err != nil {
h.logger().Printf("stream write error: %s", err)
}
return
}
json.NewEncoder(w).Encode(sliceMaxResponse{
MaxSlices: ms,

5
roaring/assembly.go
View file

@ -14,11 +14,6 @@
package roaring
func hasAsm() bool
func BSFQ(memory uint64) int
func POPCNTQ(memory uint64) int
// bit population count, take from
// https://code.google.com/p/go/issues/detail?id=4988#c11

7
roaring/assembly_asm.go
View file

@ -16,6 +16,13 @@
package roaring
func hasAsm() bool
func BSFQ(memory uint64) int
func POPCNTQ(memory uint64) int
//go:noescape
var useAsm = hasAsm()

3
roaring/assembly_generic.go
View file

@ -16,6 +16,9 @@
package roaring
func hasAsm() bool {return false}
func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) }
func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) }
func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) }

15
roaring/roaring.go
View file

@ -904,9 +904,17 @@ func (c *container) bitmapCountRange(start, end uint32) int {
var n uint64
i, j := start/64, end/64
// Special case when start and end fall in the same word.
if i == j {
offi, offj := start%64, 64-end%64
n += popcount((c.bitmap[i] >> offi) << (offj + offi))
return int(n)
}
// Count partial starting word.
if off := start % 64; off != 0 {
n += popcount(c.bitmap[i] << off)
n += popcount(c.bitmap[i] >> off)
i++
}
// Count words in between.
@ -916,9 +924,8 @@ func (c *container) bitmapCountRange(start, end uint32) int {
// Count partial ending word.
if int(j) < len(c.bitmap) {
if off := end % 64; off != 0 {
n += popcount(c.bitmap[j] >> off)
}
off := 64 - (end % 64)
n += popcount(c.bitmap[j] << off)
}
return int(n)

43
roaring/roaring_internal_test.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
import (
"testing"
)
func TestBitmapCountRange(t *testing.T) {
c := container{}
tests := []struct {
start uint32
end uint32
bitmap []uint64
exp int
}{
{start: 0, end: 1, bitmap: []uint64{1}, exp: 1},
{start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2},
{start: 67, end: 68, bitmap: []uint64{0, 0x8}, exp: 1},
{start: 1, end: 68, bitmap: []uint64{0x3, 0x8, 0xF}, exp: 2},
{start: 1, end: 258, bitmap: []uint64{0xF, 0x8, 0xA, 0x4, 0xFFFFFFFFFFFFFFFF}, exp: 9},
{start: 66, end: 71, bitmap: []uint64{0xF, 0xFFFFFFFFFFFFFF18}, exp: 2},
{start: 63, end: 64, bitmap: []uint64{0x8000000000000000}, exp: 1},
}
for i, test := range tests {
c.bitmap = test.bitmap
if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp {
t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
}
}
}

4
webui/index.html
View file

@ -6,11 +6,11 @@
<link rel="stylesheet" href="/assets/style.css">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Pilosa WebUI</title>
<link rel="icon" type="image/png" href="https://dc3kpxyuw05cb.cloudfront.net/img/favicon.png">
<link rel="icon" type="image/png" href="https://www.pilosa.com/img/favicon.png">
</head>
<body>
<div class="header">
<img src="https://dc3kpxyuw05cb.cloudfront.net/img/logo.svg" width="110" alt="">
<img src="https://www.pilosa.com/img/logo.svg" width="110" alt="">
<div id="server-version" class=""></div>
</div>
<div class="container">