From fbae0f4d15bec9a0e5c24e68bf402cf1a0f6b394 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 15 May 2017 11:31:41 -0500 Subject: [PATCH 01/18] asm build options for non amd64 --- roaring/assembly.go | 5 ----- roaring/assembly_asm.go | 7 +++++++ roaring/assembly_generic.go | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/roaring/assembly.go b/roaring/assembly.go index 1e5d49519..ea67cf66e 100644 --- a/roaring/assembly.go +++ b/roaring/assembly.go @@ -14,11 +14,6 @@ package roaring -func hasAsm() bool - -func BSFQ(memory uint64) int - -func POPCNTQ(memory uint64) int // bit population count, take from // https://code.google.com/p/go/issues/detail?id=4988#c11 diff --git a/roaring/assembly_asm.go b/roaring/assembly_asm.go index eec8234b8..3eb182cce 100644 --- a/roaring/assembly_asm.go +++ b/roaring/assembly_asm.go @@ -16,6 +16,13 @@ package roaring +func hasAsm() bool + +func BSFQ(memory uint64) int + +func POPCNTQ(memory uint64) int + + //go:noescape var useAsm = hasAsm() diff --git a/roaring/assembly_generic.go b/roaring/assembly_generic.go index 746970ba0..997bfdaa3 100644 --- a/roaring/assembly_generic.go +++ b/roaring/assembly_generic.go @@ -16,6 +16,9 @@ package roaring +func hasAsm() bool {return false} + + func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) } func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) } func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) } From f39285313417a713fd0a5230ddd51ccc9d2bf77a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 16 May 2017 11:11:11 -0500 Subject: [PATCH 02/18] update sliceMaxByIndex by view --- client.go | 9 ++++++++- cmd/backup.go | 6 +++--- ctl/export.go | 8 +++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index 058d6d836..6a4e29641 100644 --- a/client.go +++ b/client.go @@ -474,7 +474,14 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view s tw := tar.NewWriter(w) // Find the maximum number of slices. - maxSlices, err := c.MaxSliceByIndex(ctx) + var maxSlices map[string]uint64 + var err error + if view == ViewStandard { + maxSlices, err = c.MaxSliceByIndex(ctx) + } else if view == ViewInverse { + maxSlices, err = c.MaxInverseSliceByIndex(ctx) + } + if err != nil { return fmt.Errorf("slice n: %s", err) } diff --git a/cmd/backup.go b/cmd/backup.go index 548781ca6..1dd3a50d6 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -43,9 +43,9 @@ Backs up the view from across the cluster into a single file. } flags := backupCmd.Flags() flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup into.") - flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.") - flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.") + flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.") + flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.") + flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.") flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") return backupCmd diff --git a/ctl/export.go b/ctl/export.go index 7da310623..fa6277f16 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -79,7 +79,13 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - maxSlices, err := client.MaxSliceByIndex(ctx) + var maxSlices map[string]uint64 + if cmd.View == pilosa.ViewStandard { + maxSlices, err = client.MaxSliceByIndex(ctx) + } else if cmd.View == pilosa.ViewInverse { + maxSlices, err = client.MaxInverseSliceByIndex(ctx) + } + if err != nil { return err } From e31beb809bffb0159a59ad45c14b096069a628e2 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 16 May 2017 12:26:25 -0500 Subject: [PATCH 03/18] fix bug in `handleGetSliceMax` that was causing protobuf request to return json --- handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/handler.go b/handler.go index 3e837bef3..c4a98d04c 100644 --- a/handler.go +++ b/handler.go @@ -258,6 +258,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { } else if _, err := w.Write(buf); err != nil { h.logger().Printf("stream write error: %s", err) } + return } json.NewEncoder(w).Encode(sliceMaxResponse{ MaxSlices: ms, From 81621e07b20855e31c9caebb65afba3e321b2702 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 16 May 2017 13:12:08 -0500 Subject: [PATCH 04/18] add tests --- client.go | 2 ++ client_test.go | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/client.go b/client.go index 6a4e29641..161a2ac2f 100644 --- a/client.go +++ b/client.go @@ -480,6 +480,8 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view s maxSlices, err = c.MaxSliceByIndex(ctx) } else if view == ViewInverse { maxSlices, err = c.MaxInverseSliceByIndex(ctx) + } else { + return ErrInvalidView } if err != nil { diff --git a/client_test.go b/client_test.go index 6b5f43195..d2716c711 100644 --- a/client_test.go +++ b/client_test.go @@ -333,6 +333,87 @@ func TestClient_BackupRestore(t *testing.T) { } } +// Ensure client backup and restore a frame with inverse view. +func TestClient_BackupInverseView(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + frameOpts := pilosa.FrameOptions{ + InverseEnabled: true, + } + frame, err := idx.CreateFrameIfNotExists("f", frameOpts) + if err != nil { + panic(err) + } + v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) + if err != nil { + panic(err) + } + f, err := v.CreateFragmentIfNotExists(0) + if err != nil { + panic(err) + } + + f.SetBit(100, 1) + f.SetBit(100, 2) + f.SetBit(100, 3) + f.SetBit(100, SliceWidth-1) + + s := NewServer() + defer s.Close() + s.Handler.Host = s.Host() + s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Holder = hldr.Holder + + c := MustNewClient(s.Host()) + + // Backup from frame. + var buf bytes.Buffer + if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil { + t.Fatal(err) + } + + // Restore to a different frame. + if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal(err) + } + if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil { + t.Fatal(err) + } + + // Verify data. + if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + t.Fatalf("unexpected bits(0): %+v", a) + } + +} + +// backup returns error with invalid view +func TestClient_BackupInvalidView(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + + s := NewServer() + defer s.Close() + s.Handler.Host = s.Host() + s.Handler.Cluster = NewCluster(1) + s.Handler.Cluster.Nodes[0].Host = s.Host() + s.Handler.Holder = hldr.Holder + + c := MustNewClient(s.Host()) + + // Backup from frame. + var buf bytes.Buffer + err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view") + if err != pilosa.ErrInvalidView { + t.Fatal(err) + } +} + // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { hldr := MustOpenHolder() From cb2c47a727da06ece512fa57a45418e326b8d454 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 17 May 2017 08:11:38 -0500 Subject: [PATCH 05/18] Move docs into main repo --- docs/administration.md | 99 ++++++++++ docs/api-reference.md | 245 ++++++++++++++++++++++++ docs/client-libraries.md | 226 ++++++++++++++++++++++ docs/configuration.md | 169 +++++++++++++++++ docs/data-model.md | 91 +++++++++ docs/faq.md | 40 ++++ docs/getting-started.md | 149 +++++++++++++++ docs/glossary.md | 57 ++++++ docs/installation.md | 349 ++++++++++++++++++++++++++++++++++ docs/introduction.md | 17 ++ docs/pdk.md | 62 ++++++ docs/query-language.md | 395 +++++++++++++++++++++++++++++++++++++++ docs/tutorials.md | 342 +++++++++++++++++++++++++++++++++ docs/webui.md | 36 ++++ 14 files changed, 2277 insertions(+) create mode 100644 docs/administration.md create mode 100644 docs/api-reference.md create mode 100644 docs/client-libraries.md create mode 100644 docs/configuration.md create mode 100644 docs/data-model.md create mode 100644 docs/faq.md create mode 100644 docs/getting-started.md create mode 100644 docs/glossary.md create mode 100644 docs/installation.md create mode 100644 docs/introduction.md create mode 100644 docs/pdk.md create mode 100644 docs/query-language.md create mode 100644 docs/tutorials.md create mode 100644 docs/webui.md diff --git a/docs/administration.md b/docs/administration.md new file mode 100644 index 000000000..9859b862c --- /dev/null +++ b/docs/administration.md @@ -0,0 +1,99 @@ ++++ +title = "Administration Guide" ++++ + +## Administration Guide + +#### Installing in production + +##### Hardware + +Pilosa is a standalone, compiled Go application, so there is no need to worry about running and configuring a Java VM. Pilosa can run on very small machines and works well with even a medium sized dataset on a personal laptop. If you are reading this section, you are likely ready to deploy a cluster of Pilosa servers handling very large datasets or high velocity data. These are guidelines for running a cluster; specific needs may differ. + +##### Memory + +Pilosa holds all row/column bitmap data in main memory. While this data is compressed more than a typical database, available memory is a primary concern. In a production environment, we recommend choosing hardware with a large amount of memory >= 64GB. Prefer a small number of hosts with lots of memory per host over a larger number with less memory each. Larger clusters tend to be less efficient overall due to increased inter-node communication. + +##### CPUs + +Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the slice, so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload. + +##### Disk + +Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs--especially if you have a write heavy application. + +##### Network + +Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an Indexing service there already should be a system of record, or ability to rebuild a Cluster quickly from backups. + +##### Overview + +While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. + +#### Importing and Exporting Data + +##### Importing + +The import API expects a csv of RowID,ColumnID's. + +When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use `pilosa sort CSV_FILE` to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. +``` +pilosa import -d project -f stargazer project-stargazer.csv +``` + +##### Exporting + +Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. +``` +curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \ + --header "Accept: text/csv" +``` + +#### Versioning + +Pilosa follows [Semantic Versioning](http://semver.org/). + +MAJOR.MINOR.PATCH: + +* MAJOR version when you make incompatible API changes, +* MINOR version when you add functionality in a backwards-compatible manner, and +* PATCH version when you make backwards-compatible bug fixes. + +##### PQL versioning + +The Pilosa server should support PQL versioning using HTTP headers. On each request, the client should send a Content-Type header and an Accept header. The server should respond with a Content-Type header that matches the client Accept header. The server should also optionally respond with a Warning header if a PQL version is in a deprecation period, or an HTTP 400 error if a PQL version is no longer supported. + +##### Upgrading + +When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. + +#### Backup/restore + +Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. + +Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node. + +For larger datasets and to make this process faster you could copy the relevant data files from the other nodes to the new one before startup. + +Note: This will only work when the replication factor is >= 2 + +##### Using Index Sync + +- Shutdown the cluster. +- Modify config file to replace existing node address with new node. +- Restart all nodes in the cluster. +- Wait for auto Index sync to replicate data from existing nodes to new node. + +##### Copying data files manually + +- To accomplish this goal you will 1st need: + - List of all Indexes on your cluster + - List of all frames in your Indexes + - Max slice per Index, listed in the /status endpoint +- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice +- Using the list of slices owned by this node you will then need to manually: + - setup a directory structure similar to the other nodes with a path for each Index/Frame + - copy each owned slice for an existing node to this new node +- Modify the cluster config file to replace the previous node address with the new node address. +- Restart the cluster +- Wait for the 1st sync (10 minutes) to validate Index connections diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 000000000..ef89e686a --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,245 @@ ++++ +title = "API Reference" ++++ + + +## API Reference + +#### `/index` + +##### `GET` + +Returns the schema of all indexes in JSON. + +Request: +``` +curl -XGET localhost:10101/index +``` + +Response: +``` +{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} +``` + +#### `/index/` + +##### `GET` + +Returns the schema of the specified index in JSON. + +Request: +``` +curl -XGET localhost:10101/index/user +``` + +Response: +``` +{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} +``` + +##### `POST` + +Creates an index with the given name. + +The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: + +* `columnLabel` (string): column label of the index. + +Request: +``` +curl localhost:10101/index/user \ + -X POST \ + -d '{"options": {"columnLabel": "user_id"}}' +``` + +Response: +``` +{} +``` + +##### `DELETE` + +Removes the given index. + +Request: +``` +curl -XDELETE localhost:10101/index/user +``` + +Response: +``` +{} +``` + +#### `/index//query` + +##### `POST` + +Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. + +Request: +``` +curl localhost:10101/index/user/query \ + -X POST \ + -d 'Bitmap(frame="language", id=5)' +``` + +Response: +``` +{"results":[{"attrs":{},"bits":[100]}]} +``` + +In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`. + +The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`. + +Request: +``` +curl localhost:10101/index/user/query?columnAttrs=true \ + -X POST \ + -d 'Bitmap(frame="language", id=5)' +``` +Response: +``` +{ + "results":[{"attrs":{},"bits":[100]}], + "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] +} +``` + +#### `/index//time-quantum` + +##### `PATCH` + +Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database. + +The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: + +* (Empty string) +* Y: year +* M: month +* D: day +* H: hour +* YM: year and month +* MD: month and day +* DH: day and hour +* YMD: year, month and day +* MDH: month, day and hour +* YMDH: year, month, day and hour + +Request: +``` +curl localhost:10101/index/user/time-quantum \ + -X POST \ + -d '{"timeQuantum": "YM"}' +``` + +Response: +``` +{} +``` + +#### `/index//frame/` + +##### `POST` + +Creates a frame in the given index with the given name. + +The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: + +* `rowLabel` (string): Row label of the frame. +* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. +* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. +* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. + +Request: +``` +curl localhost:10101/index/user/frame/language \ + -X POST \ + -d '{"options": {"rowLabel": "language_id"}}' +``` + +Response: +``` +{} +``` + +##### `DELETE` + +Removes the given frame. + +Request: +``` +curl -XDELETE localhost:10101/index/user/frame/language +``` + +Response: +``` +{} +``` + +#### `/index//frame//time-quantum` + +##### `PATCH` + +Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. + +The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: + +* (Empty string) +* Y: year +* M: month +* D: day +* H: hour +* YM: year and month +* MD: month and day +* DH: day and hour +* YMD: year, month and day +* MDH: month, day and hour +* YMDH: year, month, day and hour + +Request: +``` +curl localhost:10101/index/user/frame/language/time-quantum \ + -X POST \ + -d '{"timeQuantum": "YM"}' +``` + +Response: +``` +{} +``` + +#### `/hosts` + +##### `GET` + +Returns the hosts in the cluster. + +Request: +``` +curl -XGET localhost:10101/hosts +``` + +Response: +``` +[{"host":":10101","internalHost":""}] +``` + +#### `/version` + +##### `GET` + +Returns the version of the Pilosa server. + +Request: +``` +curl -XGET localhost:10101/version +``` + +Response: +``` +{"version":"v0.3.0-353-ge633247"} +``` + diff --git a/docs/client-libraries.md b/docs/client-libraries.md new file mode 100644 index 000000000..ab62e2bdf --- /dev/null +++ b/docs/client-libraries.md @@ -0,0 +1,226 @@ ++++ +title = "Client Libraries" ++++ + +## Client Libraries + + +#### Go + +You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-client-pilosa). Check out its [README](https://github.com/pilosa/go-client-pilosa/blob/master/README.md) for more information and installation instructions. + +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. + +Error handling has been omitted in the example below for brevity. + +```go +package startrace + +import ( + "fmt" + + pilosa "github.com/pilosa/go-client-pilosa" +) + +func main() { + // Let's create Index and Frame objects, which will contain the settings + // for the corresponding indexes and frames. + repositoryOptions, := &pilosa.ColumnOptions{ColumnLabel: "repo_id"} + repository, _ := pilosa.NewIndex("repository", repositoryOptions) + + stargazerOptions := &pilosa.RowOptions{RowLabel: "stargazer_id"} + stargazer, _ := repository.Frame("stargazer", stargazerOptions) + + languageOptions := &pilosa.RowOptions{RowLabel: "language_id"} + language, _ := repository.Frame("language", languageOptions) + + // We will just use the default client which assumes the server is at http://localhost:10101 + client := pilosa.DefaultClient() + + var response *pilosa.QueryResponse + var result *pilosa.QueryResult + + // Which repositories did user 8 star: + response, _ = client.Query(stargazer.Bitmap(8), nil) + result = response.Result() + if result != nil { + fmt.Println("User 8 starred: ", result.Bitmap.Bits) + } + + // What are the top 5 languages in the sample data: + response, _ = client.Query(language.TopN(5), nil) + if result != nil { + fmt.Println("Top 5 languages: ", result.Bitmap.Bits) + } + + // Which repositories were starred by user 8 and 18: + response, _ = client.Query( + repository.Intersect( + stargazer.Bitmap(8), + stargazer.Bitmap(18)), + nil) + result = response.Result() + if result != nil { + fmt.Println("Repositories starred by both user 8 and 18: ", result.Bitmap.Bits) + } + + // Which repositories were starred by user 8 and 18 and also were written in language 1 + response, _ = client.Query( + repository.Intersect( + stargazer.Bitmap(8), + stargazer.Bitmap(18), + language.Bitmap(1)), + nil) + result = response.Result() + if result != nil { + fmt.Println("Repositories starred by both user 8 and 18 and are in language 1: ", result.Bitmap.Bits) + } + + // Set user 99999 as a stargazer for repository 77777: + _, err = client.Query(stargazer.SetBit(99999, 77777), nil) + if err != nil { + fmt.Println("Error setting bit: ", err) + } +} +``` + +#### Python + +You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.rst) for more information and installation instructions. + +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. + +Error handling has been omitted in the example below for brevity. + +```python +from pilosa import Index, Client, PilosaError + +# Let's create Index and Frame objects, which will contain the settings +# for the corresponding indexes and frames. +repository = Index("repository", column_label="repo_id") +stargazer = repository.frame("stargazer", row_label="stargazer_id") +language = repository.frame("language", row_label="language_id") + +# We will just use the default client which assumes the server is at http://localhost:10101 +client = Client() + +# Which repositories did user 8 star: +response = client.query(stargazer.bitmap(8)) +if response.result: + print("User 8 starred: ", result.bitmap.bits) + +# What are the top 5 languages in the sample data: +response = client.query(language.topn(5)) +if response.result: + print("Top 5 languages: ", result.bitmap.bits) + +# Which repositories were starred by user 8 and 18: +response = client.query( + repository.intersect( + stargazer.bitmap(8), + stargazer.bitmap(18))) +if response.result: + print("Repositories starred by both user 8 and 18: ", result.bitmap.bits) + +# Which repositories were starred by user 8 and 18 and also were written in language 1 +response = client.query( + repository.intersect( + stargazer.bitmap(8), + stargazer.bitmap(18), + language.bitmap(1))) +if response.result: + print("Repositories starred by both user 8 and 18 and are in language 1: ", result.bitmap.bits) + +# Set user 99999 as a stargazer for repository 77777 +try: + client.query(stargazer.setbit(99999, 77777)) +except PilosaError as ex: + print("Error setting bit: ", ex) + +``` + +#### Java + +You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. + +We are going to use the index you have created in the [Getting Started](../getting-started) section. Before carrying on, make sure that example index is created and Pilosa server is running on the default address: `http://localhost:10101`. + +Error handling has been omitted in the example below for brevity. + +```java +import com.pilosa.client.*; +import com.pilosa.client.orm.*; + +public class StarTrace { + public static void main(String[] args) { + // Let's create Index and Frame objects, which will contain the settings + // for the corresponding indexes and frames. + IndexOptions repositoryOptions = IndexOptions.builder() + .setColumnLabel("repo_id") + .build(); + Index repository = Index.withName("repository", repositoryOptions); + + FrameOptions stargazerOptions = FrameOptions.builder() + .setRowLabel("stargazer_id") + .build(); + Frame stargazer = repository.frame("stargazer", stargazerOptions); + + FrameOptions languageOptions = FrameOptions.builder() + .setRowLabel("language_id") + .build(); + Frame language = repository.frame("language", languageOptions); + + // We will just use the default client which assumes the server is at http://localhost:10101 + PilosaClient client = PilosaClient.defaultClient(); + + QueryResponse response; + QueryResult result; + + // Which repositories did user 8 star: + response = client.query(stargazer.bitmap(8)); + result = response.getResult(); + if (result != null) { + System.out.println("User 8 starred: " + result.getBitmap().getBits()); + } + + // What are the top 5 languages in the sample data: + response = client.query(language.topN(5)); + result = response.getResult(); + if (result != null) { + System.out.println("Top 5 languages: " + result.getBitmap().getBits()); + } + + // Which repositories were starred by user 8 and 18: + response = client.query( + repository.intersect( + stargazer.bitmap(8), + stargazer.bitmap(18))); + result = response.getResult(); + if (result != null) { + System.out.println("Repositories starred by both user 8 and 18: " + + result.getBitmap().getBits()); + } + + // Which repositories were starred by user 8 and 18 and also were written in language 1 + response = client.query( + repository.intersect( + stargazer.bitmap(8), + stargazer.bitmap(18), + language.bitmap(1))); + result = response.getResult(); + if (result != null) { + System.out.println("Repositories starred by both user 8 and 18 and are in language 1: " + + result.getBitmap().getBits()); + } + + // Set user 99999 as a stargazer for repository 77777: + try { + client.query(stargazer.setBit(99999, 77777)) + } + catch (PilosaException ex) { + System.out.println("Error setting bit: " + ex) + } + + } +} +``` diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..f79a98253 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,169 @@ ++++ +title = "Configuration" ++++ + +## Configuration + +Pilosa can be configured through command line flags, environment variables, and/or a configuration file; configured options take precedence in that order. So if an option is specified in a command line flag, it will take precedence over the same option specified in the environment, which would take precedence over that same option specified in the configuration file. + +All options are available in all three configuration types with the exception of the `--config` option which specifies the location of the config file, and therefore will not be used if it is present in the config file. + +The syntax for each option is slightly different between each of the configuration types, but follows a simple formula. See the following three sections for an explanation of each configuration type. + +#### Command line flags + +Pilosa uses GNU/POSIX style flags. Most flags you specify as `--flagname=value` although some have a short form that is a single character and can be specified with a single dash like `-f value`. Running `pilosa server --help` will give an overview of the available flags as well as their short forms (if applicable). + +#### Environment variables + +Every command line flag has a corresponding environment variable. The environment variable is the flag name in all caps, prefxed by `PILOSA_`, and with any dashes replaced by underscores. For example: `--flag-name` becomes `PILOSA_FLAG_NAME`. + +#### Config file + +The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file: +```toml +[cluster] + poll-interval = "2m0s" + replicas = 1 +``` + +Any flag that has a value that is a comma separated list on the command line becomes an array in toml. For example `--cluster.hosts=one.pilosa.com:10101,two.pilosa.com:10101` becomes: +```toml +[cluster] + hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"] +``` + +#### All Options + +##### Anti Entropy Interval + +* Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync. +* Flag: `--anti-entropy.interval="10m0s"` +* Env: `PILOSA_ANTI_ENTROPY.INTERVAL="10m0s"` +* Config: + + ```toml + [anti-entropy] + interval = "10m0s" + ``` + +##### Bind + +* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. +* Flag: `--bind="localhost:10101"` +* Env: `PILOSA_BIND="localhost:10101"` +* Config: + + ```toml + bind = localhost:10101 + ``` + +##### Cluster Hosts + +* Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. +* Flag: `--cluster.hosts="localhost:10101"` +* Env: `PILOSA_CLUSTER.HOSTS="localhost:10101"` +* Config: + + ```toml + [cluster] + hosts = ["localhost:10101"] + ``` + +##### Cluster Internal Hosts + +* Description: List of hosts in the cluster used for internal communication. Multiple hosts should be comma separated in the flag and env forms. +* Flag: `--cluster.internal-hosts="localhost:11101"` +* Env: `PILOSA_CLUSTER.INTERNAL_HOSTS="localhost:11101"` +* Config: + + ```toml + [cluster] + internal-hosts = ["localhost:11101"] + ``` + +##### Cluster Internal Port + +* Description: Port to which Pilosa should bind for internal communication. +* Flag: `--cluster.internal-port=11101` +* Env: `PILOSA_CLUSTER.INTERNAL_PORT=11101` +* Config: + + ```toml + [cluster] + internal-port = 11101 + ``` + +##### Cluster Poll Interval + +* Description: Polling interval for cluster. +* Flag: `cluster.poll-interval="1m0s"` +* Env: `PILOSA_CLUSTER.POLL_INTERVAL="1m0s"` +* Config: + + ```toml + [cluster] + poll-interval = "1m0s" + ``` + +##### Cluster Replicas + +* Description: Number of hosts each piece of data should be stored on. +* Flag: `cluster.replicas=1` +* Env: `PILOSA_CLUSTER.REPLICAS=1` +* Config: + + ```toml + [cluster] + replicas = 1 + ``` + +##### Cluster Type + +* Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]. + * static - Messaging between nodes is disabled. This is primarily used for testing. + * http - Messages are transmitted over HTTP. + * gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip. +* Flag: `cluster.type="gossip"` +* Env: `PILOSA_CLUSTER.TYPE="gossip"` +* Config: + + ```toml + [cluster] + type = "gossip" + ``` + +##### Data Dir + +* Description: Directory to store Pilosa data files. +* Flag: `--data-dir="~/.pilosa"` +* Env: `PILOSA_DATA_DIR="~/.pilosa"` +* Config: + + ```toml + data-dir = "~/.pilosa" + ``` + +##### Profile CPU + +* Description: If this is set to a path, collect a cpu profile and store it there. +* Flag: `--profile.cpu="/path/to/somewhere"` +* Env: `PILOSA_PROFILE.CPU="/path/to/somewhere"` +* Config: + + ```toml + [profile] + cpu = "/path/to/somewhere" + ``` + +##### Profile CPU Time + +* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set. +* Flag: `--profile.cpu-time="30s"` +* Env: `PILOSA_PROFILE.CPU_TIME="30s" +* Config: + + ```toml + [profile] + cpu-time = "30s" + ``` diff --git a/docs/data-model.md b/docs/data-model.md new file mode 100644 index 000000000..f11d1da88 --- /dev/null +++ b/docs/data-model.md @@ -0,0 +1,91 @@ ++++ +title = "Data Model" ++++ + +## Data Model + +#### Overview + +The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit - if the bit is set, it indicates that a relationship exists between that particular row and column. + +Rows and columns can represent anything (they could even represent the same set of things). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix. + +Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa also has the ability to categorize rows into different "frames" and quickly retrieve the top rows in a frame sorted by the number of bits set in each row. + +![data model diagram](/img/docs/data-model.svg) + +#### Index + +The purpose of the Index is to represent a data namespace. You cannot perform cross-index queries. Column-level attributes are global to the Index. + +#### Column + +Column ids are sequential increasing integers and are common to all Frames within an Index. + +#### Row + +Row ids are sequential increasing integers namespaced to each Frame within an Index. + +#### Frame + +Frames are used to segment and define different functional characteristics within your entire index. You can think of a Frame as a table-like data partition within your Index. + +Row attributes are namespaced at the Frame level. + +##### Ranked + +Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. + +![ranked frame diagram](/img/docs/frame-ranked.svg) + +##### LRU + +The LRU cache maintains the most recently accessed Rows. + +![lru frame diagram](/img/docs/frame-lru.svg) + +#### Time Quantum + +Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. + +#### Attribute + +Attributes are arbitrary key/value pairs that can be associated to both rows or columns. This metadata is stored in a separate BoltDB data structure. + +#### Slice + +Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. + +Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. + +#### View + +Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. + +##### Standard + +The standard View contains the same Row/Column format as the input data. + +##### Inverse + +The Inverse View contains the same data with the Row and Column swapped. + +For example, the following `SetBit()` queries will result in the data described in the illustration below: +``` +SetBit(frame="A", rowID=8, columnID=3) +SetBit(frame="A", rowID=11, columnID=3) +SetBit(frame="A", rowID=19, columnID=5) +``` + +![inverse frame diagram](/img/docs/frame-inverse.svg) + +##### Time Quantums + +If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: + +``` +SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00") +SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") +``` + +![time quantum frame diagram](/img/docs/frame-time-quantum.svg) diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 000000000..89c99f890 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,40 @@ ++++ +title = "FAQ" ++++ + +## FAQ + +#### What is Pilosa? + +Pilosa is an in-memory, distributed index that is layered over persistent storage. It supports fast ad-hoc queries and segmentation. Pilosa does not require the underlying data to be moved, rather it can be populated in conjunction with data writes, or it can be backfilled asynchronously from any other data store or event processing system. This allows Pilosa to support sub-second queries against very large underlying data sets. + +#### Is Pilosa a database? + +Pilosa is not a database in the traditional sense. While Pilosa does store data (both in-memory as well as persisted to disk), it wouldn't typically be used as a primary data store. Instead, one would likely use Pilosa as an index of the data stored in a traditional database or in a data warehouse. + +#### Where does Pilosa fit in my stack? + +Pilosa sits on top of a data store or multiple data stores. +How is Pilosa different than Elasticsearch since they are both indexes? +Elasticsearch is a search engine based on Lucene, and is therefore very good at indexing and searching large volumes of unstructured text. As it matures, Elasticsearch has continued to move into the analytics space, but its core data object is still the "document". Pilosa is specifically designed to index structured data and improve query speed. By representing data as the relationship between objects, and then storing those relationships in bitmaps, Pilosa can very efficiently search and compare many millions of data points while still maintaining a small memory footprint. + +#### How do I get my data into Pilosa? + +There are typically two methods for getting data into Pilosa: importing large batches of data from an existing data set, and continuously updating Pilosa as data is added or updated. + +In the first case, one would use the `pilosa import` command to bulk load structured data into Pilosa. In order to improve this process, one can use the Pilosa Development Kit (PDK) to map structured data in the original data set onto the Pilosa schema. + +For the case where data is continually mutating, one would apply a parallel data writer at the point at which data is written to the persistent data store. This new writer would simultaneously write to Pilosa. An example use case would be one where Kafka was employed as the message broker in your data pipeline, you could introduce an additional Kafka consumer to read from the message log and write mutated data to Pilosa. + +#### What languages can I use with it? + +There is currently client support for Go, Python, and Java. If you want to use Pilosa with a different language, you can access Pilosa via the Pilosa API. + +#### Do you query Pilosa using SQL? + +One can access Pilosa directly via the terminal using the Pilosa Query Language (PQL), but a typical implementation would use one of the Pilosa client libraries to integrate with an existing codebase. There is currently client support for Go, Python, and Java. + + +#### Replication on each node? + +Pilosa supports a replication factor greater than or equal to one. When replication is configured to be greater than one, then all mutations will be replicated to additional nodes in the cluster. For example, in a five-node cluster consisting of nodes A-B-C-D-E and with replication factor of three, then a write to node B will result in data being written to nodes B, C, and D. If the replication factor is greater than the number of nodes in the cluster, the data will be replicated to every node in the cluster only once. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..fa01dd563 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,149 @@ ++++ +title = "Getting Started" ++++ + +## Getting Started + +Pilosa supports an HTTP interface which uses JSON by default. +Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html). + +> Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. + +#### Starting Pilosa + +Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa. +Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): +``` +pilosa server +``` +If you are using the Docker image, you can run an ephemeral Pilosa container on the default address using the following command: +``` +docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest +``` + +Let's make sure Pilosa is running: +``` +curl localhost:10101/status +``` + +Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` + +#### Sample Project + +In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. + +Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation. + +##### Create the Schema + +Note: +The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: +``` +curl localhost:10101/schema +{"indexes":null} +``` + +Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: +``` +curl localhost:10101/index/repository \ + -X POST \ + -d '{"options": {"columnLabel": "repo_id"}}' +``` + +Repository IDs are the main focus of the `repository` index, so we chose `repo_id` as the column label. + +Let's create the `stargazer` frame which has user IDs of stargazers as its rows: +``` +curl localhost:10101/index/repository/frame/stargazer \ + -X POST \ + -d '{"options": {"rowLabel": "stargazer_id", + "timeQuantum": "YMD", + "inverseEnabled": true}}' +``` + +Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. + +We set `inverseEnabled` to `true` in order to allow queries over columns as well as rows. + +Next up is the `language` frame, which will contain IDs for programming languages: +``` +curl localhost:10101/index/repository/frame/language \ + -X POST \ + -d '{"options": {"rowLabel": "language_id", + "inverseEnabled": true}}' +``` +##### Import Some Data + +The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download the `stargazer.csv` and `language.csv` files in that repo. + +``` +curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv +curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv +``` + +Run the following commands to import the data into Pilosa: + +``` +pilosa import -i repository -f stargazer stargazer.csv +pilosa import -i repository -f language language.csv +``` + +If you are using a Docker container for Pilosa (with name `pilosa`), you should instead copy the `*.csv` file into the container and then import them: +``` +docker cp stargazer.csv pilosa:/stargazer.csv +docker exec -it pilosa /pilosa import -i repository -f stargazer /stargazer.csv +docker cp language.csv pilosa:/language.csv +docker exec -it pilosa /pilosa import -i repository -f language /language.csv +``` + +Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. + +##### Make Some Queries + +> Note The Pilosa server comes with a [WebUI](../webui/) for constructing queries in a browser. [localhost:10101](http://localhost:10101) + +Which repositories did user 14 star: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'Bitmap(frame="stargazer", stargazer_id=14)' +``` + +What are the top 5 languages in the sample data: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'TopN(frame="language", n=5)' +``` + +Which repositories were starred by user 14 and 19: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'Intersect(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19))' +``` + +Which repositories were starred by user 14 or 19: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'Union(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19))' +``` + +Which repositories were starred by user 14 and 19 and also were written in language 1: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'Intersect(Bitmap(frame="stargazer", stargazer_id=14), Bitmap(frame="stargazer", stargazer_id=19), Bitmap(frame="language", language_id=1))' +``` + +Set user 99999 as a stargazer for repository 77777: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'SetBit(frame="stargazer", repo_id=77777, stargazer_id=99999)' +``` + +#### What's Next? + +You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 000000000..650ac8ddf --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,57 @@ ++++ +title = "Glossary" ++++ + +# Glossary + + +Index: Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes. + +Column: Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all Frames within a Index. + +Row: Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each Frame within a Index. + +Bit: A bit is the intersection of a Row and Column. + +Bitmap: The on-disk and in-memory representation of a Row. + +Roaring Bitmap: [Roaring Bitmap](http://roaringbitmap.org) is the compressed bitmap format which Pilosa uses. + +Attribute: Attributes can be associated to both rows and columns. This metadata is kept separately from the core binary matrix in a BoltDB store. + +PQL: Pilosa Query Language + +Index: The Index represents a data namespace. + +Frame: Frames are used to segment rows into different categories - row ids are namespaced by frame such that the same row id in a different frame refers to a different row. For Ranked frames, rows are kept in sorted order within the frame. + +View: Views separate the different data layouts within a Frame. The two primary views are Standard and Inverse which represent the typical row/column data and its inverse respectively. Time based Frame Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. + +Fragment: A Fragment is the intersection of a frame and slice in an index. + +Slice: Columns are sharded on a preset width. Each shard is referred to as a Slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a consistent hash. + +SliceWidth: This is the default number of columns in a slice. + +MaxSlice: The total number of slices allocated to handle current set of columns. This value is important for all nodes to efficiently distribute queries. + +Anti-entropy: A periodic process that compares each slice and its replicas across the cluster to repair inconsistencies. + +Node: An individual running instance of Pilosa server which belongs to a cluster. + +Cluster: A cluster consists of one or more nodes which share a cluster configuration. The cluster also defines how data is replicated throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. + +TopN: Given a Frame and/or RowID this query returns the ordered set of RowID's by the number of columns that have a bit set in that row. + +Tanimoto: Used for similarity queries on Pilosa data. The Tanimoto Coefficient is the ratio of the intersecting set to the union set as the measure of similarity. + +Protobuf:: [Protocol Buffers](https://developers.google.com/protocol-buffers/) is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. + +TOML: We use [TOML](https://github.com/toml-lang/toml) for our configuration file format. + +Jump Consistent Hash: A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. +https://arxiv.org/pdf/1406.2294v1.pdf + +Partition: The consistent hash is compiled with a maximum number of partitions or locations on the unit circle that keys are mapped to. Partitions are then evenly mapped to physical nodes. To add nodes to the cluster you simply need to remap the partitions, and associated data across the new cluster topography. + +Replica: A copy of a [fragment] on a different host from the original. The "cluster.replicas" configuration parameter determines how many replicas of a fragment exist in the cluster (including the original, so a value of 1 means no extra copies are made). diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..b65fb5338 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,349 @@ ++++ +title = "Installation" ++++ + + +## Installation + +Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#installing-on-linux). + +#### Installing on MacOS + +There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker. + +##### Download the Binary + +1. Download the latest release: + ``` + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-darwin-amd64.tar.gz + ``` + + Other releases can be downloaded from our Releases page on Github. + +2. Extract the binary: + ``` + tar xfz pilosa-v0.3.1-darwin-amd64.tar.gz + ``` + +3. Move the binary into your PATH so you can run `pilosa` from any shell: + ``` + cp -i pilosa-v0.3.1-darwin-amd64/pilosa /usr/local/bin + ``` + +4. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at http://pilosa.com/docs + + Version: v0.3.0-279-gcf7082f + Build Time: 2017-04-21T15:36:08+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the default configuration. + export Export data from pilosa. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! + +##### Build from Source + +1. Install the prerequisites: + + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Git](https://git-scm.com/) + * [Glide](http://glide.sh/) + +2. Clone the repo: + ``` + go get -d github.com/pilosa/pilosa + ``` + +3. Build the Pilosa repo: + ``` + cd $GOPATH/src/github.com/pilosa/pilosa + make install + ``` + +4. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at http://pilosa.com/docs + + Version: v0.3.0-279-gcf7082f + Build Time: 2017-04-21T15:36:08+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the default configuration. + export Export data from pilosa. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! + +##### Use Docker + +1. Install Docker for Mac. + +2. Confirm that the Docker daemon is running in the background: + ``` + docker version + ``` + +If you don't see the server listed, start the Docker application. + +3. Pull the official Pilosa image from Docker Hub: + ``` + docker pull pilosa/pilosa:latest + ``` + +4. Make sure Pilosa is installed successfully: + ``` + docker run --rm pilosa/pilosa:latest help + ``` + +##### What's next? + +Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index. + + +#### Installing on Linux + +There are three ways to install Pilosa on Linux: download the binary (recommended), build from source, or use Docker. + +##### Download the Binary + +1. To install the latest version of Pilosa, download the latest release: + ``` + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.3.1/pilosa-v0.3.1-linux-amd64.tar.gz + ``` + + Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. + +2. Extract the binary: + ``` + tar xfz pilosa-v0.3.1-linux-amd64.tar.gz + ``` + +3. Move the binary into your PATH so you can run `pilosa` from any shell: + ``` + cp -i pilosa-v0.3.1-linux-amd64/pilosa /usr/local/bin + ``` + +4. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at http://pilosa.com/docs + + Version: v0.3.0-279-gcf7082f + Build Time: 2017-04-21T15:36:08+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the default configuration. + export Export data from pilosa. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! + +##### Build from Source + +1. Install the prerequisites: + + * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described here (https://golang.org/doc/code.html#GOPATH). + * [Git](https://git-scm.com/) + * [Glide](http://glide.sh/) + +2. Clone the repo: + ``` + go get -d github.com/pilosa/pilosa + ``` + +3. Build the Pilosa repo: + ``` + cd $GOPATH/src/github.com/pilosa/pilosa + make install + ``` + +4. Make sure Pilosa is installed successfully: + ``` + pilosa + ``` + + If you see something like: + ``` + Pilosa is a fast index to turbocharge your database. + + This binary contains Pilosa itself, as well as common + tools for administering pilosa, importing/exporting data, + backing up, and more. Complete documentation is available + at http://pilosa.com/docs + + Version: v0.3.0-279-gcf7082f + Build Time: 2017-04-21T15:36:08+0000 + + Usage: + pilosa [command] + + Available Commands: + backup Backup data from pilosa. + bench Benchmark operations. + check Do a consistency check on a pilosa data file. + config Print the default configuration. + export Export data from pilosa. + help Help about any command + import Bulk load data into pilosa. + inspect Get stats on a pilosa data file. + restore Restore data to pilosa from a backup file. + server Run Pilosa. + sort Sort import data for optimal import performance. + + Flags: + -c, --config string Configuration file to read from. + + Use "pilosa [command] --help" for more information about a command. + ``` + + You're good to go! + + +##### Use Docker + +1. Install Docker. + +2. Confirm that the Docker daemon is running in the background: + ``` + docker version + ``` + + If you don't see the server listed, start the Docker application. + +3. Pull the official Pilosa image from Docker Hub: + ``` + docker pull pilosa/pilosa:latest + ``` + +4. Make sure Pilosa is installed successfully: + ``` + docker run --rm pilosa/pilosa:latest help + ``` + +##### What's next? + +Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index. + + + diff --git a/docs/introduction.md b/docs/introduction.md new file mode 100644 index 000000000..87cd94a3c --- /dev/null +++ b/docs/introduction.md @@ -0,0 +1,17 @@ ++++ +title = "Introduction" ++++ + + +## Introduction + + +Pilosa is an open source, distributed bitmap index. + +[//]: # (TODO insert a graphic here?) + +It is designed primarly for speed and horizontal scalability. If you have data with billions of objects that can have millions of possible attributes, and you want to explore those relationships, Pilosa can help you. + +"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface. + +Once you have Pilosa [installed]({{< ref "installation.md" >}}), the [getting started]({{< ref "getting-started.md" >}}) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration. diff --git a/docs/pdk.md b/docs/pdk.md new file mode 100644 index 000000000..1a61ca61d --- /dev/null +++ b/docs/pdk.md @@ -0,0 +1,62 @@ ++++ +title = "PDK" ++++ + +## PDK + +The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly. + +The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary. + +#### Library + +##### Mapping + +Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts: + +* an enumeration of field names +* a list of parsers that are used to parse strings in the CSV to values +* a list of commonly used, named, mapper functions +* a list of ParserMappers - objects that encapsulate all of the work related to a single frame. + +This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK. + +With this definition available, the PDK tool can run the import, which consists of these steps: + +- create the index +- create all frames +- for each CSV file, read all rows +- for each CSV record: + - generate a columnID + - apply all ParserMappers, generating a list of (frame, ID) pairs + - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) + +The process is summarized in this flowchart: + +![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg) + + +Some of the simple mapper functions available with PDK include: + +* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year. +* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11]. +* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6]. +* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23]. +* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins. +* BoolMapper: Maps a boolean value to the range [0, 1]. +* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain. +* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small. +* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets. +* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket. +* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example. +* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper. + +#### Examples + +Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help ` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality. + + diff --git a/docs/query-language.md b/docs/query-language.md new file mode 100644 index 000000000..f35b88ad3 --- /dev/null +++ b/docs/query-language.md @@ -0,0 +1,395 @@ ++++ +title = "Query Language" ++++ + +## Query Language + +This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: + +``` +{"results":[...]} +``` + +There will be one item in the `results` array for each PQL query in the request. The type of each item in the array will depend on the type of query - each query in the reference below lists it's result type. + +Row and Column labels are set and frame and index creation time respectively. When the specification of a query says *row_label* or *col_label*, one should use the labels that were set while creating the index and frame. The default row label is `id`, and the default column label is `columnID`. + +#### Conventions + +* Angle Brackets `<>` denote required arguments +* Square Brackets `[]` denote optional arguments +* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`) + +##### Examples + +Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. + +The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", repo_id=10, stargazer_id=1)` against a server using curl, you would: +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'SetBit(frame="stargazer", repo_id=10, stargazer_id=1)' +``` + +#### Arguments and Types + +* `frame` The frame specifies on which Pilosa [frame]({{< ref "glossary.md#frame" >}}) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `ROW_LABEL` Pilosa allows users to set different row labels for each frame at frame creation time. The default row label is `rowID`, but one may set a more descriptive row label for their data (such as `stargazer_id`). +* `COL_LABEL` Pilosa allows users to set a different column label for each index at index creation time. The default column label is `columnID`. +* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") +* `UINT` An unsigned integer (e.g. 42839) +* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` +* `ATTR_VALUE` Can be a string, float, integer, or bool. +* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range` +* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) + +#### Write Operations + +##### SetBit + +**Spec:** + +``` +SetBit(, , , + [timestamp=TIMESTAMP]) +``` + +**Description:** + +`SetBit`, assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column. + +**Result Type:** boolean + +A return value of `true` indicates that the bit was changed to 1. + +A return value of `false` indicates that the bit was already set to 1 and nothing changed. + + +**Examples:** + +``` +SetBit(frame="stargazer", repo_id=10, stargazer_id=1) +``` + +This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10. + +SetBit also supports providing a timestamp. To write the date that a user starred a repository. +``` +SetBit(frame="stargazer", repo_id=10, stargazer_id=1, timestamp="2016-01-01T00:00") +``` + +Setting multiple bits in a single request: +``` +SetBit(frame="stargazer", repo_id=10, stargazer_id=1) SetBit(frame="stargazer", repo_id=10, stargazer_id=2) SetBit(frame="stargazer", repo_id=20, stargazer_id=1) SetBit(frame="stargazer", repo_id=30, stargazer_id=2) +``` + +##### SetRowAttrs +**Spec:** + +``` +SetRowAttrs(, , + , + [ATTR_NAME=ATTR_VALUE ...]) +``` + +**Description:** + +`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute. + +**Result Type:** null + +SetRowAttrs queries always return `null` upon success. + +**Examples:** + +``` +SetRowAttrs(frame="stargazer", stargazer_id=10, username="mrpi", active=true) +``` + +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. + +``` +SetRowAttrs(frame="stargazer", stargazer_id=10, username=null) +``` + +Delete username value for user 10. + +##### SetColumnAttrs + +**Spec:** + +``` +SetColumnAttrs(, , + , + [ATTR_NAME=ATTR_VALUE ...]) +``` + +**Description:** + +`SetColumnAttrs` associates arbitrary key/value pairs with a column in an index. + +**Result Type:** null + +SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. + +**Examples:** + +``` +SetColumnAttrs(frame="stargazer", repo_id=10, stars=123, url="http://projects.pilosa.com/10", active=true) +``` + +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap]({{< ref "query-language.md#bitmap" >}}) query like so `Bitmap(frame="stargazer", repo_id=10)`. + +``` +SetColumnAttrs(frame="stargazer", repo_id=10, url=null) +``` + +Delete url value for repo 10. + + +##### ClearBit + +**Spec:** + +``` +SetBit(, , , + [timestamp=TIMESTAMP]) +``` + +**Description:** + +`ClearBit`, assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column. + +**Result Type:** boolean + +A return value of `true` indicates that the bit was toggled from 1 to 0. + +A return value of `false` indicates that the bit was already set to 0 and nothing changed. + +**Examples:** + +``` +ClearBit(frame="stargazer", repo_id=10, stargazer_id=1) +``` + +Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame. + + +#### Read Operations + +##### Bitmap + +**Spec:** + +``` +Bitmap(, ( | =UINT)) +``` + +**Description:** + +`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row label or column label is given in the query. It also retrieves any attributes set on that row or column. + +**Result Type:** object with attrs and bits. + +e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` + +**Examples:** + +Query all repositories that user 1 has starred. +``` +Bitmap(frame="stargazer", stargazer_id=1) +``` + +Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` + +* attrs are the attributes for user 1 +* bits are the repositories which user 1 has starred. + +##### Union + +**Spec:** + +``` +Union([BITMAP_CALL ...]) +``` + +**Description:** + +Union performs a logical OR on the results of each `BITMAP_CALL` query passed to it. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query all repositories that are contributed by multiple users +``` +Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2)) +``` + +Returns `{"attrs":{},"bits":[10, 20, 30]}`. + +* bits are repositories that were starred by user 1 OR user 2 + +##### Intersect + + +**Spec:** + +``` +Intersect(, [BITMAP_CALL ...]) +``` + +**Description:** + +Intersect performs a logical AND on the results of each `BITMAP_CALL` query passed to it. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query repositories which have been starred by two users. + +``` +Intersect(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2)) +``` + +Returns `{"attrs":{},"bits":[10]}`. + +* bits are repositories that were starred by user 1 AND user 2 + +##### Difference + +**Spec:** + +``` +Difference(, [BITMAP_CALL ...]) +``` + +**Description:** + +Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query repositories which have been starred by one user and not another. +``` +Difference(Bitmap(frame="stargazer", stargazer_id=1), Bitmap( frame="stargazer", stargazer_id=2)) +``` + +Return `{"results":[{"attrs":{},"bits":[20]}]}` + +* bits are repositories that were starred by user 1 BUT NOT user 2 + +``` +Difference(Bitmap(frame="stargazer", stargazer_id=2), Bitmap( frame="stargazer", stargazer_id=1)) +``` + +Return `{"attrs":{},"bits":[30]}` + +* Bits are repositories that were starred by user 2 BUT NOT user 1 + +##### Count +**Spec:** + +``` +Count() +``` + +**Description:** + +Returns the number of set bits in the `BITMAP_CALL` passed in. + +**Result Type:** int + +**Examples:** + +Query the number of repositories to which a user has contributed. +``` +Count(Bitmap(frame="stargazer", stargazer_id=1)) +``` + +Return `2` + +* Result is the number of repositories that user 1 has starred. + +##### TopN + +**Spec:** + +``` +TopN([BITMAP_CALL], , [n=UINT], + [, ]) +``` + +**Description:** + +Return the id and count of the top `n` bitmaps (by count of bits) in the frame. +The `field` and `filters` arguments work together to only return Bitmaps which +have the attribute specified by `field` with one of the values specified in +`filters`. + +**Result Type:** array of key/count objects + +**Examples:** + +``` +TopN(frame="stargazer") +``` + +Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]` + +* key is a user +* count is amount of repositories +* Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository. + +``` +TopN(frame="stargazer", n=2) +``` + +Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]` + +* Results are the top two users sorted by number of repositories they've starred in descending order. + +``` +TopN(Bitmap(frame="language", language_id=1), frame="stargazer", n=2) +``` + +Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` + +* Results are the top two users sorted by the number of repositories that they've starred which are written in language 1. + +##### Range Queries + +**Spec:** + +``` +Range(, , + , ) +``` + +**Description:** + +Similar to `Bitmap`, but only returns bits which were set with timestamps +between the given `start` and `end` timestamps. + +**Result Type:** object with attrs and bits + + +**Examples:** + +When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range. +``` +Range(frame="stargazer", stargazer_id=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +``` + +Returns `{{"attrs":{},"bits":[10]}` + +* bits are repositories which were starred by user 1 from 2010-01-01 to 2017-03-02 diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 000000000..d5ec25433 --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,342 @@ ++++ +title = "Tutorials" ++++ + +## Tutorials + +#### Transportation + +##### Introduction + +New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. + +Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions). + +We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step. + +After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. + +##### Data Model + +The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: + +* Distance: miles, floating point +* Fare: dollars, floating point +* Number of passengers: integer +* Dropoff location: latitude and longitude, floating point +* Pickup location: latitude and longitude, floating point +* Dropoff time: timestamp +* Pickup time: timestamp + +We import these fields, creating one or more Pilosa frames from each of them: + +frame |mapping +------------|--------------------- +cab_type |direct map of enum int → row ID +dist_miles |round(dist) → row ID +total_amount_dollars |round(dist) → row ID +passenger_count |direct map of integer value → row ID +drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +drop_year |year(timestamp) → row ID +drop_month |month(timestamp) → row ID +drop_day |day(timestamp) → row ID +drop_time |time of day mapped to one of 48 half-hour buckets +pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID +pickup_year |year(timestamp) → row ID +pickup_month |month(timestamp) → row ID +pickup_day |day(timestamp) → row ID +pickup_time |time of day mapped to one of 48 half-hour buckets → row ID + +We also created two extra frames that represent the duration and average speed of each ride: + +frame |mapping +--------------------|------------- +duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID +speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID + +##### Mapping + +Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. + +###### 0 columns → 1 frame + +cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. + +###### 1 column → 1 frame + +The following three frames are mapped in a simple direct way from single columns of the original data. + +dist_miles: each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. + +In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code: +```go +lfm := pdk.LinearFloatMapper{ + Min: -0.5, + Max: 3600.5, + Res: 3601, +} +``` + +`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. + +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). +```go +pdk.BitMapper{ + Frame: "dist_miles", + Mapper: lfm, + Parsers: []pdk.Parser{pdk.FloatParser{}}, + Fields: []int{fields["trip_distance"]}, +}, +``` + +These same objects are represented in the JSON definition file: +```go +{ + "Fields": [ + "Trip_distance": 10, + ] +"Mappers": [ + { + "Name": "lfm0", + "Min": -0.5, + "Max": 3600.5, + "Res": 3600 + }, + ], + "BitMappers": [ + { + "Frame": "dist_miles", + "Mapper": { + "Name": "lfm0" + }, + "Parsers": [ + {"Name": "FloatParser"} + ], + "Fields": "Trip_distance", + } + ] +} +``` + +Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names to column indices. We use these names in the BitMapper definitions to keep things human-readable. + +**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. + +**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. + +###### 1 column → multiple frames + +When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. + +We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". + +We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". + +We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. + +###### Multiple columns → 1 frame + +The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. + +We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. + +Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. + +###### Complex mappings + +We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: +```go +durm := pdk.CustomMapper{ + Func: func(fields ...interface{}) interface{} { + start := fields[0].(time.Time) + end := fields[1].(time.Time) + return end.Sub(start).Minutes() + }, + Mapper: lfm, +} +``` + +##### Import process + +After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. + +##### Queries + +Now we can run some example queries. + +Count per cab type can be retrieved, sorted, with a single PQL call. + +``` +TopN(frame=cab_type) +``` + +High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. + +``` +TopN(frame=pickup_grid_id) +``` + +Average of total_amount per passenger_count can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. + +```python +queries = '' +pcounts = range(10) +for i in pcounts: + queries += "TopN(Bitmap(id=%d, frame='passenger_count.n'), frame=total_amount_dollars.n)" % i +resp = requests.post(qurl, data=queries) + +average_amounts = [] +for pcount, topn in zip(pcounts, resp.json()['results']): + wsum = sum([r['count'] * r['key'] for r in topn]) + count = sum([r['count'] for r in topn]) + average_amounts.append(float(wsum)/count) +``` + +For more examples and details, see this [ipython notebook](https://github.com/alanbernstein/pilosa-notebooks/blob/master/taxi-use-case.ipynb). + +#### Chemical similarity search + +##### Overview + +The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. + +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. + +Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. +``` +T(A,B)= Intersect(A,B) / (Count(A) + Count(B) - Intersect(A,B)) +``` + +A and B are sets of fingerprint bits on in the fingerprints of molecule A and molecule B. AB is the set of common bits of fingerprints of both molecule A and B. The Tanimoto coefficient ranges from 0 when the fingerprints have no bits in common, to 1 when the fingerprints are identical. + +All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase + +##### Data model + +We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. + +Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. + +Given a SMILES encoded molecule and a similarity threshold, we want to retrieve all molecule ids (or SMILES) that have a similarity percentage greater than or equal to the similarity threshold. For example, given a molecule with: +``` +SMILES = "IC=C1/CCC(C(=O)O1)c2cccc3ccccc23" +threshold = 90 +``` + +return the set of molecules that have at least a 90% similarity with the given molecule. + +The Inverse view swaps the rows and columns automatically to enable queries over either the chembl_id or fingerprint. + +Standard View is used to calculate similarity +``` +Index: mole + View: Standard + Col: chembl_id + Frame: fingerprint + Row: position_id ("on" bit positions of a fingerprint) +``` + +Inverse View is used for finding chembl_id based on given SMILES. +From a given SMILES, we use RDKit to convert it to fingerprints with "on" bit position. From "on" bit positions, we can search a list of chembl_ids that match the bit positions. To choose the right chembl_id, we need another query to Standard View then choose the right chembl_id which has the length that matches the given fingerprint's length after using RDKit to convert SMILES to fingerprint. +``` +Index: mole + View: Inverse + Col: position_id ("on" bit positions of a fingerprint) + Frame: fingerprint + Row: chembl_id +``` + +After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. + +##### Import process + +To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. + +For the `mole` index, each row in the csv file has the format 'chembl_id, position_id' by running the following command from Chem-usecase: +``` +python import_from_sdf.py -p -file id_fingerprint.csv +``` + + +First, follow the instruction in the [getting started]({{< ref "getting-started.md" >}}) guide to run a Pilosa server. Then create the indexes and frames according to the schemas outlined in the Data Model section above. +The option cacheSize should be set as amount of chembl_id to calculate effectively for the whole data set, so we need to calculate amount of chembl_id. We have total 1678393 chembl_id (it will displayed after import_from_sdf.py script running), then the cacheSize should be >= 1678393 +``` +curl localhost:10101/index/mole \ + -X POST \ + -d '{"options": {"columnLabel": "position_id"}}' + +curl localhost:10101/index/mole/frame/fingerprint \ + -X POST \ + -d '{"options": {"rowLabel": "chembl_id", "inverseEnabled": true, "cacheSize": 2000000, "cacheType": "ranked"}}' + +``` + +Run the following commands to import the csv data into the `mole` index: +``` +pilosa import -d mole -f fingerprint id_fingerprint.csv +``` + +##### Queries + +Get chembl_id from a given SMILES: +``` +python get_mol_fr_smile.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" +``` + +Return chembl_id = 6223. This script uses Pilosa’s Intersection query to get all chemlb_id that have positions are on, which following these steps: + +* Convert SMILES to fingerprint bit "on" positions + + ```python + from rdkit import Chem + from rdkit.Chem import AllChem + mol=Chem.MolFromSmiles("I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23") + fp = list(AllChem.GetMorganFingerprintAsBitVect(mol, 2, nBits=4096).GetOnBits()) + ``` + +* Query all chembl_id that have all "on" positions from the inverse view, return list of chembl_id + + ```python + bit_maps = ["Bitmap(position_id=%s, frame=%s, inversed=%s)" % (f, frame, True) for f in fp] + bitmap_string = ', '.join(bit_maps) + intersection = "Intersect(%s)" % bitmap_string + mole_ids = requests.post("http://%s/index/%s/query" % (host, db), data=intersection).json()["results"][0]["bits"] + ``` + +* From list of chembl_id, query all "on" position from mol index, if the length of array of "on" position is matched to len(fp) then return that chembl_id, otherwise the given SMILES does not exist. + + ```python + for m in mole_ids: + mol = requests.post("http://%s/index/%s/query" % (host, db), data="Bitmap(chembl_id=%s, frame=%s)" % (m, frame)).json()["results"][0]["bits"] + existed_mol = False + if len(mol) == len(fp): + found = m + existed_mol = True + break + ``` + +Retrieve molecule_ids that have similarity with SMILES="I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" and similarity threshold = 70% +``` +python similar.py -s "I\C=C/1\CCC(C(=O)O1)c2cccc3ccccc23" -t 70 +``` + +Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN query to get all chemlb_id that have position is on, which following these steps: + +* Get chembl_id from a SMILES (steps discussed above) + +* Query Pilosa’s TopN to get list of similarity chembl_id + ```python + query_string = 'TopN(Bitmap(chembl_id=6223, frame="fingerprint"), frame="fingerprint", n=2000000, tanimotoThreshold=70)' + topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) + ``` + +##### Benchmark + +To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: +``` +python benchmarks.py -id 6223 +``` + +As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. + +Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster diff --git a/docs/webui.md b/docs/webui.md new file mode 100644 index 000000000..191ab964e --- /dev/null +++ b/docs/webui.md @@ -0,0 +1,36 @@ ++++ +title = "WebUI" ++++ + +## WebUI + +The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101) +This can be used for constructing queries and viewing the cluster status. + +#### Console + +The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. + +Each query's result will be displayed in the Output section along with the query time. + +The Console will keep a record of each query and its result with the latest query on top. + +![console](/img/docs/webui-console.png) + +In addition to standard PQL, the console supports a few special commands, prefixed with `:`. + +- `:create index ` +- `:delete index ` +- `:use ` +- `:create frame ` +- `:delete frame ` + +Index and frame creation also supports options like `columnLabel`,`rowLabel` or `inverseEnabled`. When creating new index or new frame, add options by using the keys documented in [API reference](../api-reference). + +- `:create index columnLabel=col_id` +- `:create frame rowLabel=row_id inverseEnabled=true cacheSize=10000` + + +#### Cluster Admin + +Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. From fb4651cdb8092e1dfad3b0384b57d306ce351282 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 18 May 2017 12:04:37 -0500 Subject: [PATCH 06/18] fix 3 separate bugs in bitmapCountRange in order of the diff: 1. When the start and end of the range fall in the same word, special handling is needed to "mask" off the beginning and end of the word simultaneously to avoid counting bits at the beginning or end of the word that aren't in the range. 2. `i++` is needed at the end of the first partial word to avoid counting this word in the next block. 3. the shift amount for right shifts is 64 - (end % 64) rather than just end % 64. If end is (e.g.) 68, then 68 - 64 is 4 and we are only interested in the first 4 bits of the word, so we must right shift by 60 bits, not 4 bits. --- roaring/roaring.go | 10 +++++++++- roaring/roaring_internal_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 roaring/roaring_internal_test.go diff --git a/roaring/roaring.go b/roaring/roaring.go index e0810a6c2..1a4d8d486 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -904,9 +904,17 @@ func (c *container) bitmapCountRange(start, end uint32) int { var n uint64 i, j := start/64, end/64 + // Special case when start and end fall in the same word. + if i == j { + offi, offj := start%64, 64-end%64 + n += popcount((c.bitmap[i] << offi) >> (offj + offi)) + return int(n) + } + // Count partial starting word. if off := start % 64; off != 0 { n += popcount(c.bitmap[i] << off) + i++ } // Count words in between. @@ -916,7 +924,7 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Count partial ending word. if int(j) < len(c.bitmap) { - if off := end % 64; off != 0 { + if off := 64 - (end % 64); off != 0 { n += popcount(c.bitmap[j] >> off) } } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go new file mode 100644 index 000000000..17a35c5e1 --- /dev/null +++ b/roaring/roaring_internal_test.go @@ -0,0 +1,31 @@ +package roaring + +import ( + "testing" +) + +func TestBitmapCountRange(t *testing.T) { + c := container{bitmap: []uint64{1}} + cnt := c.bitmapCountRange(63, 65) + if cnt != 1 { + t.Fatalf("count of %v from 63 to 65 should be 1, but got %v", c.bitmap, cnt) + } + + c = container{bitmap: []uint64{0, 0x8000000000000000}} + cnt = c.bitmapCountRange(65, 66) + if cnt != 0 { + t.Fatalf("count of %v from 65 to 66 should be 0, but got %v", c.bitmap, cnt) + } + + c = container{bitmap: []uint64{0, 0xF000000000000000}} + cnt = c.bitmapCountRange(65, 66) + if cnt != 1 { + t.Fatalf("count of %v from 65 to 66 should be 1, but got %v", c.bitmap, cnt) + } + + c = container{bitmap: []uint64{0x1, 0xFF00000000000000}} + cnt = c.bitmapCountRange(62, 66) + if cnt != 3 { + t.Fatalf("count of %v from 62 to 66 should be 3, but got %v", c.bitmap, cnt) + } +} From b102b8cbc84b592cdeedfd671a2b449d63bf2c19 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 18 May 2017 14:45:36 -0500 Subject: [PATCH 07/18] Use "note" class instead of blockquotes. This is needed to support the blockquotes in the style guide for the website. --- docs/getting-started.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index fa01dd563..108c12737 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -7,7 +7,9 @@ title = "Getting Started" Pilosa supports an HTTP interface which uses JSON by default. Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html). -> Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. +
+

Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit.

+
#### Starting Pilosa @@ -100,7 +102,9 @@ Note that, both the user IDs and the repository IDs were remapped to sequential ##### Make Some Queries -> Note The Pilosa server comes with a [WebUI](../webui/) for constructing queries in a browser. [localhost:10101](http://localhost:10101) +
+

Note the Pilosa server comes with a WebUI for constructing queries in a browser. In local development, it is available at localhost:10101.

+
Which repositories did user 14 star: ``` From c9d585281791ab1d3eb5662dac283fbb00bd8e29 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 18 May 2017 15:47:27 -0500 Subject: [PATCH 08/18] Decrease heading depths now that H3 tags are supported by website CSS --- docs/administration.md | 32 ++++++++++++++++---------------- docs/api-reference.md | 38 +++++++++++++++++++------------------- docs/client-libraries.md | 6 +++--- docs/configuration.md | 30 +++++++++++++++--------------- docs/data-model.md | 28 ++++++++++++++-------------- docs/faq.md | 14 +++++++------- docs/getting-started.md | 16 ++++++++-------- docs/glossary.md | 3 +-- docs/installation.md | 24 ++++++++++++------------ docs/pdk.md | 8 ++++---- docs/tutorials.md | 34 +++++++++++++++++----------------- docs/webui.md | 4 ++-- 12 files changed, 118 insertions(+), 119 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 9859b862c..26c7ed7ba 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -4,35 +4,35 @@ title = "Administration Guide" ## Administration Guide -#### Installing in production +### Installing in production -##### Hardware +#### Hardware Pilosa is a standalone, compiled Go application, so there is no need to worry about running and configuring a Java VM. Pilosa can run on very small machines and works well with even a medium sized dataset on a personal laptop. If you are reading this section, you are likely ready to deploy a cluster of Pilosa servers handling very large datasets or high velocity data. These are guidelines for running a cluster; specific needs may differ. -##### Memory +#### Memory Pilosa holds all row/column bitmap data in main memory. While this data is compressed more than a typical database, available memory is a primary concern. In a production environment, we recommend choosing hardware with a large amount of memory >= 64GB. Prefer a small number of hosts with lots of memory per host over a larger number with less memory each. Larger clusters tend to be less efficient overall due to increased inter-node communication. -##### CPUs +#### CPUs Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the slice, so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload. -##### Disk +#### Disk Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs--especially if you have a write heavy application. -##### Network +#### Network Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an Indexing service there already should be a system of record, or ability to rebuild a Cluster quickly from backups. -##### Overview +#### Overview While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. -#### Importing and Exporting Data +### Importing and Exporting Data -##### Importing +#### Importing The import API expects a csv of RowID,ColumnID's. @@ -41,7 +41,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import -d project -f stargazer project-stargazer.csv ``` -##### Exporting +#### Exporting Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. ``` @@ -49,7 +49,7 @@ curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&vie --header "Accept: text/csv" ``` -#### Versioning +### Versioning Pilosa follows [Semantic Versioning](http://semver.org/). @@ -59,15 +59,15 @@ MAJOR.MINOR.PATCH: * MINOR version when you add functionality in a backwards-compatible manner, and * PATCH version when you make backwards-compatible bug fixes. -##### PQL versioning +#### PQL versioning The Pilosa server should support PQL versioning using HTTP headers. On each request, the client should send a Content-Type header and an Accept header. The server should respond with a Content-Type header that matches the client Accept header. The server should also optionally respond with a Warning header if a PQL version is in a deprecation period, or an HTTP 400 error if a PQL version is no longer supported. -##### Upgrading +#### Upgrading When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. -#### Backup/restore +### Backup/restore Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. @@ -77,14 +77,14 @@ For larger datasets and to make this process faster you could copy the relevant Note: This will only work when the replication factor is >= 2 -##### Using Index Sync +#### Using Index Sync - Shutdown the cluster. - Modify config file to replace existing node address with new node. - Restart all nodes in the cluster. - Wait for auto Index sync to replicate data from existing nodes to new node. -##### Copying data files manually +#### Copying data files manually - To accomplish this goal you will 1st need: - List of all Indexes on your cluster diff --git a/docs/api-reference.md b/docs/api-reference.md index ef89e686a..55bbde24e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -5,9 +5,9 @@ title = "API Reference" ## API Reference -#### `/index` +### `/index` -##### `GET` +#### `GET` Returns the schema of all indexes in JSON. @@ -21,9 +21,9 @@ Response: {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` -#### `/index/` +### `/index/` -##### `GET` +#### `GET` Returns the schema of the specified index in JSON. @@ -37,7 +37,7 @@ Response: {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` -##### `POST` +#### `POST` Creates an index with the given name. @@ -57,7 +57,7 @@ Response: {} ``` -##### `DELETE` +#### `DELETE` Removes the given index. @@ -71,9 +71,9 @@ Response: {} ``` -#### `/index//query` +### `/index//query` -##### `POST` +#### `POST` Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. @@ -107,9 +107,9 @@ Response: } ``` -#### `/index//time-quantum` +### `/index//time-quantum` -##### `PATCH` +#### `PATCH` Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database. @@ -139,9 +139,9 @@ Response: {} ``` -#### `/index//frame/` +### `/index//frame/` -##### `POST` +#### `POST` Creates a frame in the given index with the given name. @@ -165,7 +165,7 @@ Response: {} ``` -##### `DELETE` +#### `DELETE` Removes the given frame. @@ -179,9 +179,9 @@ Response: {} ``` -#### `/index//frame//time-quantum` +### `/index//frame//time-quantum` -##### `PATCH` +#### `PATCH` Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. @@ -211,9 +211,9 @@ Response: {} ``` -#### `/hosts` +### `/hosts` -##### `GET` +#### `GET` Returns the hosts in the cluster. @@ -227,9 +227,9 @@ Response: [{"host":":10101","internalHost":""}] ``` -#### `/version` +### `/version` -##### `GET` +#### `GET` Returns the version of the Pilosa server. diff --git a/docs/client-libraries.md b/docs/client-libraries.md index ab62e2bdf..e7a439371 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -5,7 +5,7 @@ title = "Client Libraries" ## Client Libraries -#### Go +### Go You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-client-pilosa). Check out its [README](https://github.com/pilosa/go-client-pilosa/blob/master/README.md) for more information and installation instructions. @@ -84,7 +84,7 @@ func main() { } ``` -#### Python +### Python You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.rst) for more information and installation instructions. @@ -139,7 +139,7 @@ except PilosaError as ex: ``` -#### Java +### Java You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. diff --git a/docs/configuration.md b/docs/configuration.md index f79a98253..12153dde1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,15 +10,15 @@ All options are available in all three configuration types with the exception of The syntax for each option is slightly different between each of the configuration types, but follows a simple formula. See the following three sections for an explanation of each configuration type. -#### Command line flags +### Command line flags Pilosa uses GNU/POSIX style flags. Most flags you specify as `--flagname=value` although some have a short form that is a single character and can be specified with a single dash like `-f value`. Running `pilosa server --help` will give an overview of the available flags as well as their short forms (if applicable). -#### Environment variables +### Environment variables Every command line flag has a corresponding environment variable. The environment variable is the flag name in all caps, prefxed by `PILOSA_`, and with any dashes replaced by underscores. For example: `--flag-name` becomes `PILOSA_FLAG_NAME`. -#### Config file +### Config file The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.poll-interval=2m0s` and `--cluster.replicas=1` look like this in the config file: ```toml @@ -33,9 +33,9 @@ Any flag that has a value that is a comma separated list on the command line bec hosts = ["one.pilosa.com:10101", "two.pilosa.com:10101"] ``` -#### All Options +### All Options -##### Anti Entropy Interval +#### Anti Entropy Interval * Description: Interval at which the cluster will run its anti-entropy routine which makes sure that all replicas of each fragment are in sync. * Flag: `--anti-entropy.interval="10m0s"` @@ -47,7 +47,7 @@ Any flag that has a value that is a comma separated list on the command line bec interval = "10m0s" ``` -##### Bind +#### Bind * Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. * Flag: `--bind="localhost:10101"` @@ -58,7 +58,7 @@ Any flag that has a value that is a comma separated list on the command line bec bind = localhost:10101 ``` -##### Cluster Hosts +#### Cluster Hosts * Description: List of hosts in the cluster. Multiple hosts should be comma separated in the flag and env forms. * Flag: `--cluster.hosts="localhost:10101"` @@ -70,7 +70,7 @@ Any flag that has a value that is a comma separated list on the command line bec hosts = ["localhost:10101"] ``` -##### Cluster Internal Hosts +#### Cluster Internal Hosts * Description: List of hosts in the cluster used for internal communication. Multiple hosts should be comma separated in the flag and env forms. * Flag: `--cluster.internal-hosts="localhost:11101"` @@ -82,7 +82,7 @@ Any flag that has a value that is a comma separated list on the command line bec internal-hosts = ["localhost:11101"] ``` -##### Cluster Internal Port +#### Cluster Internal Port * Description: Port to which Pilosa should bind for internal communication. * Flag: `--cluster.internal-port=11101` @@ -94,7 +94,7 @@ Any flag that has a value that is a comma separated list on the command line bec internal-port = 11101 ``` -##### Cluster Poll Interval +#### Cluster Poll Interval * Description: Polling interval for cluster. * Flag: `cluster.poll-interval="1m0s"` @@ -106,7 +106,7 @@ Any flag that has a value that is a comma separated list on the command line bec poll-interval = "1m0s" ``` -##### Cluster Replicas +#### Cluster Replicas * Description: Number of hosts each piece of data should be stored on. * Flag: `cluster.replicas=1` @@ -118,7 +118,7 @@ Any flag that has a value that is a comma separated list on the command line bec replicas = 1 ``` -##### Cluster Type +#### Cluster Type * Description: Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]. * static - Messaging between nodes is disabled. This is primarily used for testing. @@ -133,7 +133,7 @@ Any flag that has a value that is a comma separated list on the command line bec type = "gossip" ``` -##### Data Dir +#### Data Dir * Description: Directory to store Pilosa data files. * Flag: `--data-dir="~/.pilosa"` @@ -144,7 +144,7 @@ Any flag that has a value that is a comma separated list on the command line bec data-dir = "~/.pilosa" ``` -##### Profile CPU +#### Profile CPU * Description: If this is set to a path, collect a cpu profile and store it there. * Flag: `--profile.cpu="/path/to/somewhere"` @@ -156,7 +156,7 @@ Any flag that has a value that is a comma separated list on the command line bec cpu = "/path/to/somewhere" ``` -##### Profile CPU Time +#### Profile CPU Time * Description: Amount of time to collect cpu profiling data if `profile.cpu` is set. * Flag: `--profile.cpu-time="30s"` diff --git a/docs/data-model.md b/docs/data-model.md index f11d1da88..b4b98de9a 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -4,7 +4,7 @@ title = "Data Model" ## Data Model -#### Overview +### Overview The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit - if the bit is set, it indicates that a relationship exists between that particular row and column. @@ -14,59 +14,59 @@ Pilosa lays out data first in rows, so queries which get all the set bits in one ![data model diagram](/img/docs/data-model.svg) -#### Index +### Index The purpose of the Index is to represent a data namespace. You cannot perform cross-index queries. Column-level attributes are global to the Index. -#### Column +### Column Column ids are sequential increasing integers and are common to all Frames within an Index. -#### Row +### Row Row ids are sequential increasing integers namespaced to each Frame within an Index. -#### Frame +### Frame Frames are used to segment and define different functional characteristics within your entire index. You can think of a Frame as a table-like data partition within your Index. Row attributes are namespaced at the Frame level. -##### Ranked +#### Ranked Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. ![ranked frame diagram](/img/docs/frame-ranked.svg) -##### LRU +#### LRU The LRU cache maintains the most recently accessed Rows. ![lru frame diagram](/img/docs/frame-lru.svg) -#### Time Quantum +### Time Quantum Setting a time quantum on a frame creates extra indices which allow Range queries down to the interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. -#### Attribute +### Attribute Attributes are arbitrary key/value pairs that can be associated to both rows or columns. This metadata is stored in a separate BoltDB data structure. -#### Slice +### Slice Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. -#### View +### View Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. -##### Standard +#### Standard The standard View contains the same Row/Column format as the input data. -##### Inverse +#### Inverse The Inverse View contains the same data with the Row and Column swapped. @@ -79,7 +79,7 @@ SetBit(frame="A", rowID=19, columnID=5) ![inverse frame diagram](/img/docs/frame-inverse.svg) -##### Time Quantums +#### Time Quantums If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: diff --git a/docs/faq.md b/docs/faq.md index 89c99f890..cea161aa3 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -4,21 +4,21 @@ title = "FAQ" ## FAQ -#### What is Pilosa? +### What is Pilosa? Pilosa is an in-memory, distributed index that is layered over persistent storage. It supports fast ad-hoc queries and segmentation. Pilosa does not require the underlying data to be moved, rather it can be populated in conjunction with data writes, or it can be backfilled asynchronously from any other data store or event processing system. This allows Pilosa to support sub-second queries against very large underlying data sets. -#### Is Pilosa a database? +### Is Pilosa a database? Pilosa is not a database in the traditional sense. While Pilosa does store data (both in-memory as well as persisted to disk), it wouldn't typically be used as a primary data store. Instead, one would likely use Pilosa as an index of the data stored in a traditional database or in a data warehouse. -#### Where does Pilosa fit in my stack? +### Where does Pilosa fit in my stack? Pilosa sits on top of a data store or multiple data stores. How is Pilosa different than Elasticsearch since they are both indexes? Elasticsearch is a search engine based on Lucene, and is therefore very good at indexing and searching large volumes of unstructured text. As it matures, Elasticsearch has continued to move into the analytics space, but its core data object is still the "document". Pilosa is specifically designed to index structured data and improve query speed. By representing data as the relationship between objects, and then storing those relationships in bitmaps, Pilosa can very efficiently search and compare many millions of data points while still maintaining a small memory footprint. -#### How do I get my data into Pilosa? +### How do I get my data into Pilosa? There are typically two methods for getting data into Pilosa: importing large batches of data from an existing data set, and continuously updating Pilosa as data is added or updated. @@ -26,15 +26,15 @@ In the first case, one would use the `pilosa import` command to bulk load struct For the case where data is continually mutating, one would apply a parallel data writer at the point at which data is written to the persistent data store. This new writer would simultaneously write to Pilosa. An example use case would be one where Kafka was employed as the message broker in your data pipeline, you could introduce an additional Kafka consumer to read from the message log and write mutated data to Pilosa. -#### What languages can I use with it? +### What languages can I use with it? There is currently client support for Go, Python, and Java. If you want to use Pilosa with a different language, you can access Pilosa via the Pilosa API. -#### Do you query Pilosa using SQL? +### Do you query Pilosa using SQL? One can access Pilosa directly via the terminal using the Pilosa Query Language (PQL), but a typical implementation would use one of the Pilosa client libraries to integrate with an existing codebase. There is currently client support for Go, Python, and Java. -#### Replication on each node? +### Replication on each node? Pilosa supports a replication factor greater than or equal to one. When replication is configured to be greater than one, then all mutations will be replicated to additional nodes in the cluster. For example, in a five-node cluster consisting of nodes A-B-C-D-E and with replication factor of three, then a write to node B will result in data being written to nodes B, C, and D. If the replication factor is greater than the number of nodes in the cluster, the data will be replicated to every node in the cluster only once. diff --git a/docs/getting-started.md b/docs/getting-started.md index 108c12737..96fd4e9e7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -7,11 +7,11 @@ title = "Getting Started" Pilosa supports an HTTP interface which uses JSON by default. Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html). -
+

Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit.

-#### Starting Pilosa +### Starting Pilosa Follow the steps in the [Install]({{< ref "installation.md" >}}) document to install Pilosa. Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at `localhost:10101`): @@ -30,13 +30,13 @@ curl localhost:10101/status Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` -#### Sample Project +### Sample Project In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about the top 1,000 most recently updated Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model) section of the documentation. -##### Create the Schema +#### Create the Schema Note: The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: @@ -74,7 +74,7 @@ curl localhost:10101/index/repository/frame/language \ -d '{"options": {"rowLabel": "language_id", "inverseEnabled": true}}' ``` -##### Import Some Data +#### Import Some Data The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download the `stargazer.csv` and `language.csv` files in that repo. @@ -100,9 +100,9 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. -##### Make Some Queries +#### Make Some Queries -
+

Note the Pilosa server comes with a WebUI for constructing queries in a browser. In local development, it is available at localhost:10101.

@@ -148,6 +148,6 @@ curl localhost:10101/index/repository/query \ -d 'SetBit(frame="stargazer", repo_id=77777, stargazer_id=99999)' ``` -#### What's Next? +### What's Next? You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Tutorials](../tutorials/) for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/). diff --git a/docs/glossary.md b/docs/glossary.md index 650ac8ddf..e4bef02e3 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -2,8 +2,7 @@ title = "Glossary" +++ -# Glossary - +## Glossary Index: Indexes are the top level container in Pilosa - similar to a database in an RDBMS. Queries cannot operate across multiple indexes. diff --git a/docs/installation.md b/docs/installation.md index b65fb5338..c97ed4303 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -7,11 +7,11 @@ title = "Installation" Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#installing-on-linux). -#### Installing on MacOS +### Installing on MacOS There are three ways to install Pilosa on MacOS: download the binary (recommended), build from source, or use Docker. -##### Download the Binary +#### Download the Binary 1. Download the latest release: ``` @@ -71,7 +71,7 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende You're good to go! -##### Build from Source +#### Build from Source 1. Install the prerequisites: @@ -131,7 +131,7 @@ There are three ways to install Pilosa on MacOS: download the binary (recommende You're good to go! -##### Use Docker +#### Use Docker 1. Install Docker for Mac. @@ -152,16 +152,16 @@ If you don't see the server listed, start the Docker application. docker run --rm pilosa/pilosa:latest help ``` -##### What's next? +#### What's next? Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index. -#### Installing on Linux +### Installing on Linux There are three ways to install Pilosa on Linux: download the binary (recommended), build from source, or use Docker. -##### Download the Binary +#### Download the Binary 1. To install the latest version of Pilosa, download the latest release: ``` @@ -221,7 +221,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende You're good to go! -##### Build from Source +#### Build from Source 1. Install the prerequisites: @@ -282,7 +282,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende You're good to go! -##### Use Docker +#### Use Docker 1. Install Docker. @@ -303,17 +303,17 @@ There are three ways to install Pilosa on Linux: download the binary (recommende docker run --rm pilosa/pilosa:latest help ``` -##### What's next? +#### What's next? Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index. diff --git a/docs/tutorials.md b/docs/tutorials.md index d5ec25433..5b36bd62b 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -4,9 +4,9 @@ title = "Tutorials" ## Tutorials -#### Transportation +### Transportation -##### Introduction +#### Introduction New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set. @@ -16,7 +16,7 @@ We've written a tool to help import the NYC taxi data into Pilosa - this tool is After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa. -##### Data Model +#### Data Model The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at: @@ -54,15 +54,15 @@ frame |mapping duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID -##### Mapping +#### Mapping Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. -###### 0 columns → 1 frame +##### 0 columns → 1 frame cab_type: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. -###### 1 column → 1 frame +##### 1 column → 1 frame The following three frames are mapped in a simple direct way from single columns of the original data. @@ -124,7 +124,7 @@ Here, we define a list of Mappers, each including a name, which we use to refer **passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. -###### 1 column → multiple frames +##### 1 column → multiple frames When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. @@ -134,7 +134,7 @@ We might continue this pattern with hours, minutes, and seconds, but we don't ha We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. -###### Multiple columns → 1 frame +##### Multiple columns → 1 frame The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. @@ -142,7 +142,7 @@ We do all of this for each location of interest, one for pickup and one for drop Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. -###### Complex mappings +##### Complex mappings We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: ```go @@ -156,11 +156,11 @@ durm := pdk.CustomMapper{ } ``` -##### Import process +#### Import process After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. See [PDK](../pdk) for more details on this process. -##### Queries +#### Queries Now we can run some example queries. @@ -194,9 +194,9 @@ for pcount, topn in zip(pcounts, resp.json()['results']): For more examples and details, see this [ipython notebook](https://github.com/alanbernstein/pilosa-notebooks/blob/master/taxi-use-case.ipynb). -#### Chemical similarity search +### Chemical similarity search -##### Overview +#### Overview The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. @@ -211,7 +211,7 @@ A and B are sets of fingerprint bits on in the fingerprints of molecule A and mo All source code to calculate tanimoto for molecule fingerprint using Pilosa is available in a Github repository https://github.com/pilosa/chem-usecase -##### Data model +#### Data model We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. @@ -248,7 +248,7 @@ Index: mole After retrieving chembl_id from the Inverse View, we can use the Tanimoto coefficient to compare chembl_id with the entire data set of molecules. The result of this comparison is the list of `chembl_id`s that have a Tanimoto coefficient greater than the given threshold. -##### Import process +#### Import process To import data into Pilosa, we need to get chembl_id and SMILES from SD files, convert SMILES to Morgan fingerprints, and then write chembl_id and fingerprint to Pilosa. The fastest way is to extracted chembl_id and SMILES from SD file to csv file, then use the `pilosa import` command to import the csv file into Pilosa. Since chembl_id in the SD file is always paired with CHEMBL, e.g CHEMBL6329, and because Pilosa doesn't support string keys, we will ignore CHEMBL and instead use chembl_id as an integer key. @@ -276,7 +276,7 @@ Run the following commands to import the csv data into the `mole` index: pilosa import -d mole -f fingerprint id_fingerprint.csv ``` -##### Queries +#### Queries Get chembl_id from a given SMILES: ``` @@ -330,7 +330,7 @@ Return chembl_id = [6223, 269758, 6206, 6228]. This script uses Pilosa’s TopN topn = requests.post("http://127.0.0.1:10101/index/mol/query" , data=query_string) ``` -##### Benchmark +#### Benchmark To run benchmark for specific chembl_id for different similarity threshold at percentage of [50, 70, 75, 80, 85, 90], run following command: ``` diff --git a/docs/webui.md b/docs/webui.md index 191ab964e..8084cbb80 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -7,7 +7,7 @@ title = "WebUI" The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101) This can be used for constructing queries and viewing the cluster status. -#### Console +### Console The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. @@ -31,6 +31,6 @@ Index and frame creation also supports options like `columnLabel`,`rowLabel` or - `:create frame rowLabel=row_id inverseEnabled=true cacheSize=10000` -#### Cluster Admin +### Cluster Admin Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. From 21692cfddb78d472de700bcd79cf58fe282bc10a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 19 May 2017 14:17:57 -0500 Subject: [PATCH 09/18] Hide build directory in .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c924add8b..8db43b3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ default.etcd/ vendor .protoc-gen-gofast .DS_Store +build From 5788a459f88019d8788739dbd131f1cf438d5e61 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 19 May 2017 15:00:50 -0500 Subject: [PATCH 10/18] Remove .n frame suffix --- docs/tutorials.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 5b36bd62b..dc120fdd6 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -182,7 +182,7 @@ Average of total_amount per passenger_count can be computed with some postproces queries = '' pcounts = range(10) for i in pcounts: - queries += "TopN(Bitmap(id=%d, frame='passenger_count.n'), frame=total_amount_dollars.n)" % i + queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i resp = requests.post(qurl, data=queries) average_amounts = [] From c0ddbe0e3fba41573cc46ee510153ac3d6f5799a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 19 May 2017 15:15:23 -0500 Subject: [PATCH 11/18] fix bitmapCountRange and test had been thinking that index 0 was the most significant bit, but based on the bitmapAdd function, it must be the least significant bit --- roaring/roaring.go | 6 ++--- roaring/roaring_internal_test.go | 40 +++++++++++++++----------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 1a4d8d486..57825a5b2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -907,13 +907,13 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Special case when start and end fall in the same word. if i == j { offi, offj := start%64, 64-end%64 - n += popcount((c.bitmap[i] << offi) >> (offj + offi)) + n += popcount((c.bitmap[i] >> offi) << (offj + offi)) return int(n) } // Count partial starting word. if off := start % 64; off != 0 { - n += popcount(c.bitmap[i] << off) + n += popcount(c.bitmap[i] >> off) i++ } @@ -925,7 +925,7 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Count partial ending word. if int(j) < len(c.bitmap) { if off := 64 - (end % 64); off != 0 { - n += popcount(c.bitmap[j] >> off) + n += popcount(c.bitmap[j] << off) } } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 17a35c5e1..5cff4a30f 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -5,27 +5,25 @@ import ( ) func TestBitmapCountRange(t *testing.T) { - c := container{bitmap: []uint64{1}} - cnt := c.bitmapCountRange(63, 65) - if cnt != 1 { - t.Fatalf("count of %v from 63 to 65 should be 1, but got %v", c.bitmap, cnt) + c := container{} + tests := []struct { + start uint32 + end uint32 + bitmap []uint64 + exp int + }{ + {start: 0, end: 1, bitmap: []uint64{1}, exp: 1}, + {start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2}, + {start: 67, end: 68, bitmap: []uint64{0, 0x8}, exp: 1}, + {start: 1, end: 68, bitmap: []uint64{0x3, 0x8, 0xF}, exp: 2}, + {start: 1, end: 258, bitmap: []uint64{0xF, 0x8, 0xA, 0x4, 0xFFFFFFFFFFFFFFFF}, exp: 9}, + {start: 66, end: 71, bitmap: []uint64{0xF, 0xFFFFFFFFFFFFFF18}, exp: 2}, + {start: 63, end: 64, bitmap: []uint64{0x8000000000000000}, exp: 1}, } - - c = container{bitmap: []uint64{0, 0x8000000000000000}} - cnt = c.bitmapCountRange(65, 66) - if cnt != 0 { - t.Fatalf("count of %v from 65 to 66 should be 0, but got %v", c.bitmap, cnt) - } - - c = container{bitmap: []uint64{0, 0xF000000000000000}} - cnt = c.bitmapCountRange(65, 66) - if cnt != 1 { - t.Fatalf("count of %v from 65 to 66 should be 1, but got %v", c.bitmap, cnt) - } - - c = container{bitmap: []uint64{0x1, 0xFF00000000000000}} - cnt = c.bitmapCountRange(62, 66) - if cnt != 3 { - t.Fatalf("count of %v from 62 to 66 should be 3, but got %v", c.bitmap, cnt) + for i, test := range tests { + c.bitmap = test.bitmap + if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp { + t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret) + } } } From 9deee72bda5b556625351bd7d8dd75350627e671 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 19 May 2017 16:27:49 -0500 Subject: [PATCH 12/18] Add 32-bit build to release process --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f614068d4..e0fc7056c 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ crossbuild: vendor release: make crossbuild GOOS=linux GOARCH=amd64 + make crossbuild GOOS=linux GOARCH=386 make crossbuild GOOS=darwin GOARCH=amd64 install: vendor From a5918d01d9d41b6a2512d1eeba8aa7b60d8b6012 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 19 May 2017 17:09:22 -0500 Subject: [PATCH 13/18] fix offset check to make sense --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 57825a5b2..ee46622f1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -924,7 +924,7 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Count partial ending word. if int(j) < len(c.bitmap) { - if off := 64 - (end % 64); off != 0 { + if off := 64 - (end % 64); off != 64 { n += popcount(c.bitmap[j] << off) } } From 6b3b459659aedeae9fb6ba3dc9e05cf2ff0db454 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 22 May 2017 09:46:52 -0500 Subject: [PATCH 14/18] add license to roaring_internal_test --- roaring/roaring_internal_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 5cff4a30f..41d0d167e 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import ( From 788386b49acabd637d17cdee9e7b713e6fc56d72 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 10 May 2017 20:19:04 -0600 Subject: [PATCH 15/18] Implement 'config' CLI command. Renames `config` to `generate-config` and implements a new `config` command that generates the configuration file based on the current state instead of printing a static string. --- cmd/config.go | 12 ++++++-- cmd/generate_config.go | 49 ++++++++++++++++++++++++++++++++ cmd/server.go | 19 ++----------- config.go | 4 +++ ctl/config.go | 30 +++++--------------- ctl/generate_config.go | 63 ++++++++++++++++++++++++++++++++++++++++++ ctl/server.go | 42 ++++++++++++++++++++++++++++ glide.lock | 2 +- 8 files changed, 178 insertions(+), 43 deletions(-) create mode 100644 cmd/generate_config.go create mode 100644 ctl/generate_config.go create mode 100644 ctl/server.go diff --git a/cmd/config.go b/cmd/config.go index 589367712..9452cbc3c 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -22,18 +22,21 @@ import ( "github.com/spf13/cobra" "github.com/pilosa/pilosa/ctl" + "github.com/pilosa/pilosa/server" ) var Conf *ctl.ConfigCommand func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) + Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "config", - Short: "Print the default configuration.", - Long: `config prints the default configuration to stdout -`, + Short: "Print the current configuration.", + Long: `config prints the current configuration to stdout`, + RunE: func(cmd *cobra.Command, args []string) error { + Conf.Config = Server.Config if err := Conf.Run(context.Background()); err != nil { return err } @@ -41,6 +44,9 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command }, } + // Attach flags to the command. + ctl.BuildServerFlags(confCmd, Server) + return confCmd } diff --git a/cmd/generate_config.go b/cmd/generate_config.go new file mode 100644 index 000000000..b0622b64d --- /dev/null +++ b/cmd/generate_config.go @@ -0,0 +1,49 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/pilosa/pilosa/ctl" +) + +var GenerateConf *ctl.GenerateConfigCommand + +func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) + confCmd := &cobra.Command{ + Use: "generate-config", + Short: "Print the default configuration.", + Long: `generate-config prints the default configuration to stdout +`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := GenerateConf.Run(context.Background()); err != nil { + return err + } + return nil + }, + } + + return confCmd +} + +func init() { + subcommandFns["generate-config"] = NewGenerateConfigCommand +} diff --git a/cmd/server.go b/cmd/server.go index 54603806f..d282ed71f 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" + "github.com/pilosa/pilosa/ctl" "github.com/pilosa/pilosa/server" ) @@ -85,23 +86,9 @@ on the configured port.`, return nil }, } - flags := serveCmd.Flags() - flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") - flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") - flags.IntVarP(&Server.Config.MaxWritesPerRequest, "max-writes-per-request", "", Server.Config.MaxWritesPerRequest, "Number of write commands per request.") - flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") - flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? - flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") - flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") - flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") - flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") - flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") - flags.StringVarP(&Server.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&Server.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") + // Attach flags to the command. + ctl.BuildServerFlags(serveCmd, Server) return serveCmd } diff --git a/config.go b/config.go index 3e58a2d13..15e115005 100644 --- a/config.go +++ b/config.go @@ -99,3 +99,7 @@ func (d *Duration) UnmarshalText(text []byte) error { func (d Duration) MarshalText() (text []byte, err error) { return []byte(d.String()), nil } + +func (d Duration) MarshalTOML() ([]byte, error) { + return []byte(d.String()), nil +} diff --git a/ctl/config.go b/ctl/config.go index d084c1e2c..2047941c7 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -18,14 +18,15 @@ import ( "context" "fmt" "io" - "strings" + toml "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" ) // ConfigCommand represents a command for printing a default config. type ConfigCommand struct { *pilosa.CmdIO + Config *pilosa.Config } // NewConfigCommand returns a new instance of ConfigCommand. @@ -37,27 +38,10 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *ConfigCommand // Run prints out the default config. func (cmd *ConfigCommand) Run(ctx context.Context) error { - fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` -data-dir = "~/.pilosa" -bind = "localhost:10101" -max-writes-per-request = 5000 - -[cluster] - poll-interval = "2m0s" - replicas = 1 - hosts = [ - "localhost:10101", - ] - -[anti-entropy] - interval = "10m0s" - -[profile] - cpu = "" - cpu-time = "30s" - -[plugins] - path = "" -`)+"\n") + buf, err := toml.Marshal(*cmd.Config) + if err != nil { + return err + } + fmt.Fprintln(cmd.Stdout, string(buf)) return nil } diff --git a/ctl/generate_config.go b/ctl/generate_config.go new file mode 100644 index 000000000..8024a160c --- /dev/null +++ b/ctl/generate_config.go @@ -0,0 +1,63 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/pilosa/pilosa" +) + +// GenerateConfigCommand represents a command for printing a default config. +type GenerateConfigCommand struct { + *pilosa.CmdIO +} + +// NewGenerateConfigCommand returns a new instance of GenerateConfigCommand. +func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *GenerateConfigCommand { + return &GenerateConfigCommand{ + CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + } +} + +// Run prints out the default config. +func (cmd *GenerateConfigCommand) Run(ctx context.Context) error { + fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` +data-dir = "~/.pilosa" +bind = "localhost:10101" +max-writes-per-request = 5000 + +[cluster] + poll-interval = "2m0s" + replicas = 1 + hosts = [ + "localhost:10101", + ] + +[anti-entropy] + interval = "10m0s" + +[profile] + cpu = "" + cpu-time = "30s" + +[plugins] + path = "" +`)+"\n") + return nil +} diff --git a/ctl/server.go b/ctl/server.go new file mode 100644 index 000000000..7c246491a --- /dev/null +++ b/ctl/server.go @@ -0,0 +1,42 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctl + +import ( + "time" + + "github.com/pilosa/pilosa/server" + "github.com/spf13/cobra" +) + +// BuildServerFlags attaches a set of flags to the command for a server instance. +func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { + flags := cmd.Flags() + flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") + flags.StringVarP(&srv.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") + flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") + flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&srv.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") + flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? + flags.StringVarP(&srv.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") + flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") + flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") + flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") + flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") + flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") + flags.StringVarP(&srv.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") + flags.StringVarP(&srv.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") +} diff --git a/glide.lock b/glide.lock index a615eac54..63123f289 100644 --- a/glide.lock +++ b/glide.lock @@ -65,7 +65,7 @@ imports: - name: github.com/pelletier/go-buffruneio version: c37440a7cf42ac63b919c752ca73a85067e05992 - name: github.com/pelletier/go-toml - version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a + version: 23f644976aa7c724adf4aec911dadf4af17840ab - name: github.com/rakyll/statik version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f subpackages: From e90e1455db96773379ba6b74f07e4bee6dc4fe9f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 22 May 2017 10:38:40 -0500 Subject: [PATCH 16/18] remove branch and always count last container --- roaring/roaring.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ee46622f1..86d810450 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -924,9 +924,8 @@ func (c *container) bitmapCountRange(start, end uint32) int { // Count partial ending word. if int(j) < len(c.bitmap) { - if off := 64 - (end % 64); off != 64 { - n += popcount(c.bitmap[j] << off) - } + off := 64 - (end % 64) + n += popcount(c.bitmap[j] << off) } return int(n) From 124a2e62a87b1a06026d1d05d9fad153b6d4a0ae Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 22 May 2017 11:35:02 -0500 Subject: [PATCH 17/18] Remove curly braces (fails on some systems) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e0fc7056c..6d1182bac 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ pilosa: vendor crossbuild: vendor mkdir -p build/pilosa-$(IDENTIFIER) make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" - cp {LICENSE,README.md} build/pilosa-$(IDENTIFIER) + cp LICENSE README.md build/pilosa-$(IDENTIFIER) tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" From 18dacee64bf4dc0068ed0fb274f02e03e8bb2acd Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 22 May 2017 16:41:05 -0500 Subject: [PATCH 18/18] Update logo urls --- webui/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/index.html b/webui/index.html index 09e39d17a..034dba0f6 100644 --- a/webui/index.html +++ b/webui/index.html @@ -6,11 +6,11 @@ Pilosa WebUI - +
- +