diff --git a/.gitignore b/.gitignore index 86fa71248..c924add8b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,4 @@ default.etcd/ *.test vendor .protoc-gen-gofast - -/datadog/.DS_Store +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index 78c3e71f6..d5f1022d6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik GLIDE := $(shell command -v glide 2>/dev/null) +STATIK := $(shell command -v statik 2>/dev/null) PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) @@ -41,14 +42,24 @@ install: vendor .protoc-gen-gofast: vendor ifndef PROTOC - $(error "protoc is not available please install protoc from https://github.com/google/protobuf/releases") + $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases") endif go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast -generate: .protoc-gen-gofast +generate-protoc: .protoc-gen-gofast go generate github.com/pilosa/pilosa/internal +generate-statik: statik + go generate github.com/pilosa/pilosa + +generate: generate-protoc generate-statik + +statik: +ifndef STATIK + go get github.com/rakyll/statik +endif + docker: docker build -t "pilosa:$(VERSION)" \ --build-arg ldflags="-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)" . diff --git a/README.md b/README.md index acab512eb..377ea62aa 100644 --- a/README.md +++ b/README.md @@ -1,326 +1,64 @@ -# pilosa - -Pilosa is a bitmap index. + + + [![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa) +## An open source, distributed bitmap index. +- [Docs](#docs) +- [Getting Started](#getting-started) +- [Data Model](#data-model) +- [Query Language](#query-language) +- [Client Libraries](#client-drivers) +- [Get Support](#get-support) +- [Contributing](#contributing) + + +## Docs + +See our [Documentation](https://www.pilosa.com/docs/) for information about installing and working with Pilosa. + + ## Getting Started -Pilosa requires Go 1.7 or greater. +1. [Install Pilosa](https://www.pilosa.com/docs/installation/). -You can download the source by running `go get`: +2. [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration: -```sh -$ go get github.com/pilosa/pilosa -``` + ```shell + pilosa server + ``` + + and verify that it's running: + + ```shell + curl localhost:10101/nodes + ``` -Now you can install the `pilosa` binary: +3. Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities. -```sh -$ go install github.com/pilosa/pilosa/cmd/... -``` -Now run a single pilosa node with the default configuration: +## Data Model -```sh -pilosa server -``` +Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) works. -## Configuration -Running just `pilosa` will show a list of available subcommands. `pilosa help -` will show usage information and the available flags for the command. +## Query Language -Any flag can be specified at the command line, in an environment variables, -and/or in a toml config file. The environment variable for any flag is that -flag, upper cased, prefixed with `PILOSA_`, and with any dashes converted to -underscores. For example, if you would specify `--cluster.poll-interval=30s` at -the command line, you would set `PILOSA_CLUSTER.POLL_INTERVAL=30s` in the -environment. For the configuration file, a dot in a flag denotes nesting with in -the config file. See the example config file below for examples of this. +You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL). -You can specify a configuration by setting the `--config` flag when running -`pilosa`. +## Client Libraries -```sh -pilosa server --config custom-config-file.cfg -``` +There are supported libraries for the following languages: +- [Go](https://www.pilosa.com/docs/client-libraries/#go) +- [Java](https://www.pilosa.com/docs/client-libraries/#java) +- [Python](https://www.pilosa.com/docs/client-libraries/#python) -The config file uses the [TOML](https://github.com/toml-lang/toml) configuration file format, -and should look like: +## Get Support -``` -data-dir = "/tmp/pil0" -bind = "127.0.0.1:10101" +There are [several channels](https://www.pilosa.com/community/#support) availble for you to reach out to us for support. -[cluster] - poll-interval = "2m0s" - replicas = 2 - hosts = [ - "127.0.0.1:10101", - "127.0.0.1:10102", - ] +## Contributing -[anti-entropy] - interval = "10m0s" - -[profile] - cpu = "/home/mycpuprofile" - cpu-time = "30s" -``` - -You can generate a template config file with default values with: - -```sh -pilosa config -``` - -The first two configuration options will be unique to each node in the cluster: - -`data-dir`: directory in which data is stored to disk - -`bind`: IP and port that pilosa will listen on - -The remaining configuration options should be the same on every node in the cluster. - -`[cluster] replicas`: the number of replicas within the cluster - -`[cluster] hosts`: specifies each node within the cluster - -`[cluster] poll-interval`: TODO - -`[anti-entropy] interval`: TODO - -There are also some profiling options for debugging and performance tuning - these don't need to be the same across the cluster and are mostly useful for doing Pilosa development. - -`[profile] cpu`: Path at which to store cpu profiling data which will be taken when pilosa starts. - -`[profile] cpu-time`: Amount of time for which to collect cpu profiling data at startup. - -## Docker - -You can create a Pilosa container using `make docker` or equivalently: -``` -docker build -t pilosa:latest . -``` - -You can run a temporary container using: -``` -docker run -it --rm --name pilosa -p 10101:10101 pilosa:latest -``` - -When you click `Ctrl+C` to stop the container, the container and the data in the container will be erased. You can leave out `--rm` flag to keep the data in the container. See [Docker documentation](https://docs.docker.com) for other options. - -## Usage - -You can interact with Pilosa via HTTP requests to the host:port on which you have Pilosa running. -The following examples illustrate how to do this using `curl` with a Pilosa cluster running on -127.0.0.1 port 10101. - -Return the version of Pilosa: -```sh -$ curl "http://127.0.0.1:10101/version" -``` - -Return a list of all indexes and frames in the index: -```sh -$ curl "http://127.0.0.1:10101/schema" -``` - -### Index and Frame Schema - -Before running a query, the corresponding index and frame must be created. Note that index and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`). - -You can create the index `sample-idx` using: - -```sh -$ curl -XPOST "http://127.0.0.1:10101/index" \ - -d '{"index": "sample-idx"}' -``` - -Optionally, you can specify the column label on index creation: - -```sh -$ curl -XPOST "http://127.0.0.1:10101/index" \ - -d '{"index": "sample-idx", "options": {"columnLabel": "user"}}' -``` - -The frame `collaboration` may be created using the following call: - -```sh -$ curl -XPOST "http://127.0.0.1:10101/frame" \ - -d '{"index": "sample-idx", "frame": "collaboration"}' -``` - -It is possible to specify the frame row label on frame creation: - -```sh -$ curl -XPOST "http://127.0.0.1:10101/frame" \ - -d '{"index": "sample-idx", "frame": "collaboration", "options": {"rowLabel": "project"}}' -``` - -### Queries - -Queries to Pilosa require sending a POST request where the query itself is sent as POST data. -You specify the index on which to perform the query with a URL argument `index=index-name`. - -In this section, we assume both the index `sample-idx` with column label `user` and the frame `collaboration` with row label `project` was created. - -A query sent to index `sample-idx` will have the following format: - -```sh -$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query()' -``` - -The `Query()` object referenced above should be made up of one or more of the query types listed below. -So for example, a SetBit() query would look like this: -```sh -$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'SetBit(project=10, frame="collaboration", user=1)' -``` - -Query results have the format `{"results":[]}`, where `results` is a list of results for each `Query()`. This -means that you can provide multiple `Query()` objects with each HTTP request and `results` will contain -the results of all of the queries. - -```sh -$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query() Query() Query()' -``` - ---- -#### SetBit() -``` -SetBit(project=10, frame="collaboration", user=1) -``` -A return value of `{"results":[true]}` indicates that the bit was toggled from 0 to 1. -A return value of `{"results":[false]}` indicates that the bit was already set to 1 and therefore nothing changed. - -SetBit accepts an optional `timestamp` field: -``` -SetBit(project=10, frame="collaboration", user=2, timestamp="2016-12-11T10:09:07") -``` - ---- -#### ClearBit() -``` -ClearBit(project=10, frame="collaboration", user=1) -``` -A return value of `{"results":[true]}` indicates that the bit was toggled from 1 to 0. -A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed. - ---- -#### SetRowAttrs() -``` -SetRowAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true) -``` -Returns `{"results":[null]}` - ---- -#### SetColumnAttrs() ---- -``` -SetColumnAttrs(user=10, friends=123, username="mrpi", active=true) -``` - -Returns `{"results":[null]}` - ---- -#### Bitmap() -``` -Bitmap(project=10, frame="collaboration") -``` -Returns `{"results":[{"attrs":{"stars":123, "url":"http://projects.pilosa.com/10", "active":true},"bits":[1,2]}]}` where `attrs` are the -attributes set using `SetRowAttrs()` and `bits` are the bits set using `SetBit()`. - -In order to return column attributes attached to the columns of a bitmap, add `&columnAttrs=true` to the query string. Sample response: -``` -{"results":[{"attrs":{},"bits":[10]}],"columnAttrs":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]} -``` - ---- -#### Union() -``` -Union(Bitmap(project=10, frame="collaboration"), Bitmap(project=20, frame="collaboration"))) -``` -Returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[1,2]}]}`. -Note that a `Union()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`. - ---- -#### Intersect() -``` -Intersect(Bitmap(project=10, frame="collaboration"), Bitmap(project=20, frame="collaboration"))) -``` -Returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[1]}]}`. -Note that an `Intersect()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`. - ---- -#### Difference() -``` -Difference(Bitmap(project=10, frame="collaboration"), Bitmap(project=20, frame="collaboration"))) -``` -`Difference()` represents all of the bits that are set in the first `Bitmap()` but are not set in the second `Bitmap()`. It returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[2]}]}`. -Note that a `Difference()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`. - ---- -#### Count() -``` -Count(Bitmap(project=10, frame="collaboration")) -``` -Returns the count of the number of bits set in `Bitmap()`: `{"results":[28]}` - ---- -#### Range() -``` -Range(project=10, frame="collaboration", start="1970-01-01T00:00", end="2000-01-02T03:04") -``` - ---- -#### TopN() -``` -TopN(frame="geo") -``` -Returns all Bitmaps in the cache from frame `geo` sorted by the count of bits. - -``` -TopN(frame="geo", n=20) -``` -Returns the top 20 Bitmaps from frame `geo`. - -``` -TopN(Bitmap(project=10, frame="collaboration"), frame="geo", n=20) -``` -Returns the top 20 Bitmaps from `geo` sorted by the count of bits in the intersection with `Bitmap(project=10)`. - -``` -TopN(Bitmap(project=10, frame="collaboration"), frame="geo", n=20, field="category", [81,82]) -``` -Returns the top 20 Bitmaps from `geo`in attribute `category` with values `81 or -82` sorted by the count of bits in the intersection with `Bitmap(project=10)`. - -## Development - -### Updating dependencies - -To update dependencies, you'll need to install [Glide][]. - -Then add the new dependencies in your project: - -```sh -$ glide get github.com/foo/bar -``` - -### Protobuf - -If you update protobuf (pilosa/internal/internal.proto), then you need to run `go generate` -```sh -$ go generate -``` - -### Version - -In order to set the version number, compile Pilosa with the following argument: -```sh -$ go install --ldflags="-X main.Version=1.0.0" -``` - -[Glide]: http://glide.sh/ +Pilosa is an open source project. Please see our [Contributing Guide](https://www.pilosa.com/docs/contributing/) for information about how to get involved. diff --git a/attr.go b/attr.go index 3859a1dc7..2e45aadf8 100644 --- a/attr.go +++ b/attr.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/attr_test.go b/attr_test.go index cfc31d54b..dda280214 100644 --- a/attr_test.go +++ b/attr_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/bitmap.go b/bitmap.go index ba1a6af93..ecdb4c076 100644 --- a/bitmap.go +++ b/bitmap.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa // #cgo CFLAGS:-mpopcnt diff --git a/broadcast.go b/broadcast.go index 9e04d04f6..95a6d58f5 100644 --- a/broadcast.go +++ b/broadcast.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -17,23 +31,27 @@ type NodeSet interface { Open() error } -// StaticNodeSet represents a basic NodeSet for testing +// StaticNodeSet represents a basic NodeSet for testing. type StaticNodeSet struct { nodes []*Node } +// NewStaticNodeSet creates a statically defined NodeSet. func NewStaticNodeSet() *StaticNodeSet { return &StaticNodeSet{} } +// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. func (s *StaticNodeSet) Nodes() []*Node { return s.nodes } +// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing. func (s *StaticNodeSet) Open() error { return nil } +// Join sets the NodeSet nodes to the slice of Nodes passed in. func (s *StaticNodeSet) Join(nodes []*Node) error { s.nodes = nodes return nil @@ -49,9 +67,9 @@ func init() { NopBroadcaster = &nopBroadcaster{} } +// NopBroadcaster represents a Broadcaster that doesn't do anything. var NopBroadcaster Broadcaster -// nopBroadcaster represents a Broadcaster that doesn't do anything. type nopBroadcaster struct{} // SendSync A no-op implemenetation of Broadcaster SendSync method. @@ -85,8 +103,10 @@ type nopBroadcastReceiver struct{} func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } +// NopBroadcastReceiver is a no-op implementation of the BroadcastReceiver. var NopBroadcastReceiver = &nopBroadcastReceiver{} +// Broadcast message types. const ( MessageTypeCreateSlice = 1 MessageTypeCreateIndex = 2 @@ -95,6 +115,7 @@ const ( MessageTypeDeleteFrame = 5 ) +// MarshalMessage encodes the protobuf message into a byte slice. func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { @@ -118,6 +139,7 @@ func MarshalMessage(m proto.Message) ([]byte, error) { return append([]byte{typ}, buf...), nil } +// UnmarshalMessage decodes the byte slice into a protobuf message. func UnmarshalMessage(buf []byte) (proto.Message, error) { typ, buf := buf[0], buf[1:] diff --git a/broadcast_test.go b/broadcast_test.go index de38239d4..8f0245c88 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/cache.go b/cache.go index 5dc4d3188..e0c8047d8 100644 --- a/cache.go +++ b/cache.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -58,6 +72,7 @@ func NewLRUCache(maxEntries uint32) *LRUCache { return c } +// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion. func (c *LRUCache) BulkAdd(id, n uint64) { c.Add(id, n) } @@ -211,6 +226,7 @@ func (c *RankCache) Invalidate() { c.invalidate() } +// Recalculate rebuilds the cache. func (c *RankCache) Recalculate() { c.mu.Lock() defer c.mu.Unlock() @@ -328,19 +344,23 @@ type PairHeap struct { Pairs } +// Less implemets the Sort interface. +// reports whether the element with index i should sort before the element with index j. func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count } -func (h *Pairs) Push(x interface{}) { +// Push appends the element onto the Pair slice. +func (p *Pairs) Push(x interface{}) { // Push and Pop use pointer receivers because they modify the slice's length, // not just its contents. - *h = append(*h, x.(Pair)) + *p = append(*p, x.(Pair)) } -func (h *Pairs) Pop() interface{} { - old := *h +// Pop removes the minimum element from the Pair slice. +func (p *Pairs) Pop() interface{} { + old := *p n := len(old) x := old[n-1] - *h = old[0 : n-1] + *p = old[0 : n-1] return x } diff --git a/client.go b/client.go index 7606369ae..96e725876 100644 --- a/client.go +++ b/client.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -315,6 +329,7 @@ func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, return nil } +// MarshalImportPayload marshalls the import parameters into a protobuf byte slice. func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() @@ -982,36 +997,36 @@ func (p Bits) Less(i, j int) bool { } // RowIDs returns a slice of all the row IDs. -func (a Bits) RowIDs() []uint64 { - other := make([]uint64, len(a)) - for i := range a { - other[i] = a[i].RowID +func (p Bits) RowIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].RowID } return other } // ColumnIDs returns a slice of all the column IDs. -func (a Bits) ColumnIDs() []uint64 { - other := make([]uint64, len(a)) - for i := range a { - other[i] = a[i].ColumnID +func (p Bits) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID } return other } // Timestamps returns a slice of all the timestamps. -func (a Bits) Timestamps() []int64 { - other := make([]int64, len(a)) - for i := range a { - other[i] = a[i].Timestamp +func (p Bits) Timestamps() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Timestamp } return other } // GroupBySlice returns a map of bits by slice. -func (a Bits) GroupBySlice() map[uint64][]Bit { +func (p Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) - for _, bit := range a { + for _, bit := range p { slice := bit.ColumnID / SliceWidth m[slice] = append(m[slice], bit) } diff --git a/client_test.go b/client_test.go index 70676e399..a89a41cce 100644 --- a/client_test.go +++ b/client_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/cluster.go b/cluster.go index 4c6dc2b82..0ab8aea06 100644 --- a/cluster.go +++ b/cluster.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -13,8 +27,10 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 +) - // NodeState represents node state returned in /status endpoint for a node in the cluster. +// NodeState represents node state returned in /status endpoint for a node in the cluster. +const ( NodeStateUp = "UP" NodeStateDown = "DOWN" ) @@ -125,7 +141,7 @@ func NewCluster() *Cluster { } } -// NodeSetHosts returns the list of host strings for NodeSet members +// NodeSetHosts returns the list of host strings for NodeSet members. func (c *Cluster) NodeSetHosts() []string { if c.NodeSet == nil { return []string{} @@ -152,7 +168,7 @@ func (c *Cluster) NodeStates() map[string]string { return h } -// State returns the internal ClusterState representation. +// Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ Nodes: encodeClusterStatus(c.Nodes), @@ -229,11 +245,10 @@ func (c *Cluster) OwnsSlices(index string, maxSlice uint64, host string) []uint6 for i := uint64(0); i <= maxSlice; i++ { p := c.Partition(index, i) // Determine primary owner node. - node_index := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[node_index].Host == host { + nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) + if c.Nodes[nodeIndex].Host == host { slices = append(slices, i) } - } return slices } diff --git a/cluster_test.go b/cluster_test.go index 4c96cd5d9..3caea3fe2 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -118,6 +132,16 @@ func TestCluster_NodeStates(t *testing.T) { } } +// Ensure OwnsSlices can find the actual slice list for node and index +func TestCluster_OwnsSlices(t *testing.T) { + c := NewCluster(5) + slices := c.OwnsSlices("test", 10, "host2") + + if !reflect.DeepEqual(slices, []uint64{1, 3, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() diff --git a/cmd.go b/cmd.go index 7ff0a877a..d97f7150b 100644 --- a/cmd.go +++ b/cmd.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import "io" diff --git a/cmd/backup.go b/cmd/backup.go index d25b968f4..548781ca6 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/backup_test.go b/cmd/backup_test.go index 4e8f8fe43..58d9bf21c 100644 --- a/cmd/backup_test.go +++ b/cmd/backup_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/bench.go b/cmd/bench.go index 2dd0d86fc..0ea32d22a 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/bench_test.go b/cmd/bench_test.go index db47829ea..4b94d9392 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/check.go b/cmd/check.go index 914d499fd..56ae13265 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/check_test.go b/cmd/check_test.go index c65e38aa4..a50b48330 100644 --- a/cmd/check_test.go +++ b/cmd/check_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/config.go b/cmd/config.go index 8191d5972..589367712 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/doc.go b/cmd/doc.go index f1e0f23e6..17a54043a 100644 --- a/cmd/doc.go +++ b/cmd/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Package cmd contains all the pilosa subcommand definitions (1 per file). diff --git a/cmd/export.go b/cmd/export.go index 186759909..310e3f05c 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/export_test.go b/cmd/export_test.go index 45ddc58c6..2b30116c6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/import.go b/cmd/import.go index 43b26e7f8..6fd182f17 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/import_test.go b/cmd/import_test.go index 0251d1894..8117d46fa 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/inspect.go b/cmd/inspect.go index b3d19d013..e8526a414 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/inspect_test.go b/cmd/inspect_test.go index 619f6fbe9..4c2b64d76 100644 --- a/cmd/inspect_test.go +++ b/cmd/inspect_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index 1947288c7..648b41866 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* This is the entrypoint for the Pilosa binary. */ diff --git a/cmd/restore.go b/cmd/restore.go index 38a1a3145..80f0bdd74 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/restore_test.go b/cmd/restore_test.go index 6f8b5c15b..cdad8ca92 100644 --- a/cmd/restore_test.go +++ b/cmd/restore_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/root.go b/cmd/root.go index 7cee0e77f..d50dd05b4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( @@ -30,7 +44,7 @@ func NewRootCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { This binary contains Pilosa itself, as well as common tools for administering pilosa, importing/exporting data, backing up, and more. Complete documentation is available -at http://pilosa.com/docs +at https://www.pilosa.com/docs/ Version: ` + Version + ` Build Time: ` + BuildTime + "\n", diff --git a/cmd/root_test.go b/cmd/root_test.go index d59deb2e6..16bc82a8a 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/server.go b/cmd/server.go index f18ee4e69..833e567bb 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/server_test.go b/cmd/server_test.go index edd61c601..c2c50f996 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/cmd/sort.go b/cmd/sort.go index 668206070..e74f6cc76 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/sort_test.go b/cmd/sort_test.go index dbc5e8972..f95332886 100644 --- a/cmd/sort_test.go +++ b/cmd/sort_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd_test import ( diff --git a/config.go b/config.go index 646c77773..ebaf47e33 100644 --- a/config.go +++ b/config.go @@ -1,12 +1,32 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import "time" const ( - // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultClusterType = "static" + // DefaultHost is the default hostname to use. + DefaultHost = "localhost" + + // DefaultPort is the default port use with the hostname. + DefaultPort = "10101" + + // DefaultClusterType sets the node intercommunication method. + DefaultClusterType = "static" + + // DefaultInternalPort the port the nodes intercommunicate on. DefaultInternalPort = "14000" DefaultMetrics = "nop" ) @@ -75,6 +95,7 @@ func (d *Duration) UnmarshalText(text []byte) error { return nil } +// MarshalText writes duration value in text format. func (d Duration) MarshalText() (text []byte, err error) { return []byte(d.String()), nil } diff --git a/ctl/backup.go b/ctl/backup.go index 369455e7c..7b8465637 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/bench.go b/ctl/bench.go index b63d9165f..9e7b5d386 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/check.go b/ctl/check.go index 40be60499..34b7033fc 100644 --- a/ctl/check.go +++ b/ctl/check.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/config.go b/ctl/config.go index 886240b7b..6de94bf8c 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/doc.go b/ctl/doc.go index 1cc3981c2..65623d1f7 100644 --- a/ctl/doc.go +++ b/ctl/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // package ctl contains all pilosa subcommands other than 'server'. These are // generally administration, testing, and debugging tools. package ctl diff --git a/ctl/export.go b/ctl/export.go index 6bbf1ac0e..8f67b7896 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/import.go b/ctl/import.go index cefe2e45b..55337d980 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/inspect.go b/ctl/inspect.go index fa99ba464..f5825c35a 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/restore.go b/ctl/restore.go index ecfcf5f5a..676e81331 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/ctl/sort.go b/ctl/sort.go index 5590d2538..2d28e02c4 100644 --- a/ctl/sort.go +++ b/ctl/sort.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/datadog/datadog.go b/datadog/datadog.go index d4cf1df65..5f260106f 100644 --- a/datadog/datadog.go +++ b/datadog/datadog.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package datadog import ( diff --git a/datadog/datadog_test.go b/datadog/datadog_test.go index b5a8bf5ee..5de60165e 100644 --- a/datadog/datadog_test.go +++ b/datadog/datadog_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package datadog_test import ( diff --git a/doc.go b/doc.go index e2265bef6..9afa1a532 100644 --- a/doc.go +++ b/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Package pilosa implements the core of the Pilosa distributed bitmap index. It diff --git a/executor.go b/executor.go index e828100b8..a2c5ab8f1 100644 --- a/executor.go +++ b/executor.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -56,17 +70,41 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic opt = &ExecOptions{} } + // Don't bother calculating slices for query types that don't require it. + needsSlices := needsSlices(q.Calls) + + // MaxSlice can differ between inverse and standard views, so we need + // to send queries to different slices based on orientation. + var inverseSlices []uint64 + rowLabel := DefaultRowLabel + columnLabel := DefaultColumnLabel + // If slices aren't specified, then include all of them. if len(slices) == 0 { - if needsSlices(q.Calls) { + // Determine slices and inverseSlices for use in e.executeCall(). + if needsSlices { // Round up the number of slices. maxSlice := e.Holder.Index(index).MaxSlice() + maxInverseSlice := e.Holder.Index(index).MaxInverseSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) for i := range slices { slices[i] = uint64(i) } + + // Generate a slices of all inverse slices. + inverseSlices = make([]uint64, maxInverseSlice+1) + for i := range inverseSlices { + inverseSlices[i] = uint64(i) + } + + // Fetch column label from index. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + columnLabel = idx.ColumnLabel() } } @@ -78,6 +116,25 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { + + if call.SupportsInverse() && needsSlices { + // Fetch frame & row label based on argument. + frame, _ := call.Args["frame"].(string) + if frame == "" { + frame = DefaultFrame + } + f := e.Holder.Frame(index, frame) + if f == nil { + return nil, ErrFrameNotFound + } + rowLabel = f.RowLabel() + + // If this call is to an inverse frame send to a different list of slices. + if call.IsInverse(rowLabel, columnLabel) { + slices = inverseSlices + } + } + v, err := e.executeCall(ctx, index, call, slices, opt) if err != nil { return nil, err @@ -762,7 +819,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetRowAttrs() row field '%v' required.", rowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel) } // Copy args and remove reserved fields. @@ -822,7 +879,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if err != nil { return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { - return nil, fmt.Errorf("SetRowAttrs row field '%v' required.", rowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) } // Copy args and remove reserved fields. diff --git a/executor_test.go b/executor_test.go index 8fed950b4..084511fb0 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -27,9 +41,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -37,7 +51,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -58,9 +72,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -89,7 +103,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -121,7 +135,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -151,7 +165,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -181,7 +195,7 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -199,7 +213,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -210,7 +224,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -235,16 +249,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(hldr.Holder, NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } @@ -362,7 +376,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -409,7 +423,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { t.Fatal(err) } e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -448,7 +462,7 @@ func TestExecutor_Execute_Range(t *testing.T) { f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -468,7 +482,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", id=10)` { + } else if query.String() != `Bitmap(frame="f", rowID=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -490,7 +504,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) @@ -518,7 +532,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -540,7 +554,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", id=10)` { + } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -557,7 +571,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -585,7 +599,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(columnID=2, frame="f", rowID=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -604,7 +618,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } diff --git a/fragment.go b/fragment.go index 092f8bd8d..1d8df780f 100644 --- a/fragment.go +++ b/fragment.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -1447,10 +1461,10 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&buf, "SetBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&buf, "ClearBit(frame=%q, rowID=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) } // Verify sync is not prematurely closing. diff --git a/fragment_test.go b/fragment_test.go index f4d3fc247..5f3da53e2 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/frame.go b/frame.go index f6af6e80c..6afff11de 100644 --- a/frame.go +++ b/frame.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -17,7 +31,7 @@ import ( // Default frame settings. const ( - DefaultRowLabel = "id" + DefaultRowLabel = "rowID" DefaultCacheType = CacheTypeLRU DefaultInverseEnabled = false @@ -67,7 +81,8 @@ func NewFrame(path, index, name string) (*Frame, error) { views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), - Stats: NopStatsClient, + broadcaster: NopBroadcaster, + Stats: NopStatsClient, rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, @@ -125,7 +140,7 @@ func (f *Frame) SetRowLabel(v string) error { } // Make sure rowLabel is valid name - err := ValidateName(v) + err := ValidateLabel(v) if err != nil { return err } @@ -423,6 +438,7 @@ func (f *Frame) newView(path, name string) *View { view.LogOutput = f.LogOutput view.RowAttrStore = f.rowAttrStore view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) + view.broadcaster = f.broadcaster return view } @@ -529,6 +545,9 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro inverse = []string{ViewInverse} } else { standard = ViewsByTime(ViewStandard, *timestamp, q) + // In order to match the logic of `SetBit()`, we want bits + // with timestamps to write to both time and standard views. + standard = append(standard, ViewStandard) inverse = ViewsByTime(ViewInverse, *timestamp, q) } diff --git a/frame_test.go b/frame_test.go index 669ffcc40..fc5c01570 100644 --- a/frame_test.go +++ b/frame_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -66,6 +80,88 @@ func TestFrame_NameRestriction(t *testing.T) { } } +// Ensure that frame name validation is consistent. +func TestFrame_NameValidation(t *testing.T) { + validFrameNames := []string{ + "foo", + "hyphen-ated", + "under_score", + "abc123", + "trailing_", + } + invalidFrameNames := []string{ + "", + "123abc", + "x.y", + "_foo", + "-bar", + "abc def", + "camelCase", + "UPPERCASE", + "a12345678901234567890123456789012345678901234567890123456789012345", + } + + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + for _, name := range validFrameNames { + _, err := pilosa.NewFrame(path, "i", name) + if err != nil { + t.Fatalf("unexpected frame name: %s %s", name, err) + } + } + for _, name := range invalidFrameNames { + _, err := pilosa.NewFrame(path, "i", name) + if err == nil { + t.Fatalf("expected error on frame name: %s", name) + } + } +} + +// Ensure that frame RowLable validation is consistent. +func TestFrame_RowLabelValidation(t *testing.T) { + validRowLabels := []string{ + "", + "foo", + "hyphen-ated", + "under_score", + "abc123", + "trailing_", + "camelCase", + "UPPERCASE", + } + invalidRowLabels := []string{ + "123abc", + "x.y", + "_foo", + "-bar", + "abc def", + "a12345678901234567890123456789012345678901234567890123456789012345", + } + + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + f, err := pilosa.NewFrame(path, "i", "f") + if err != nil { + t.Fatalf("unexpected frame error: %s", err) + } + + for _, label := range validRowLabels { + if err := f.SetRowLabel(label); err != nil { + t.Fatalf("unexpected row label: %s %s", label, err) + } + } + for _, label := range invalidRowLabels { + if err := f.SetRowLabel(label); err == nil { + t.Fatalf("expected error on row label: %s", label) + } + } + +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame diff --git a/glide.lock b/glide.lock index 7ba5cdab6..a615eac54 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: a6889bf5334164aa4aebd670eee45444f1bdfa1888642d290acf33e18e57f293 -updated: 2017-04-19T17:19:43.584486333-05:00 +hash: 5066da70d6f312d032800619798684a693851eb77991b3788f68e89dfaf93925 +updated: 2017-04-25T15:31:49.712775923-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 @@ -7,8 +7,6 @@ imports: version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca -- name: github.com/CAFxX/gcnotifier - version: adea3e70515666981da25214a7d3e377e4841c22 - name: github.com/DataDog/datadog-go version: 909c02b65dd8a52e8fa6072db9752a112227cf21 subpackages: @@ -48,10 +46,10 @@ imports: subpackages: - hcl/ast - hcl/parser - - hcl/token - - json/parser - hcl/scanner - hcl/strconv + - hcl/token + - json/parser - json/scanner - json/token - name: github.com/hashicorp/memberlist @@ -68,6 +66,10 @@ imports: version: c37440a7cf42ac63b919c752ca73a85067e05992 - name: github.com/pelletier/go-toml version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a +- name: github.com/rakyll/statik + version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f + subpackages: + - fs - name: github.com/satori/go.uuid version: 879c5887cd475cd7864858769793b2ceb0d44feb - name: github.com/spf13/afero @@ -103,4 +105,4 @@ imports: - unicode/norm - name: gopkg.in/yaml.v2 version: a3f3340b5840cee44f372bddb5880fcbc419b46a -devImports: [] +testImports: [] diff --git a/glide.yaml b/glide.yaml index 4b1ff6038..32421ffad 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,6 +31,10 @@ import: - package: github.com/spf13/viper - package: github.com/gorilla/mux version: ^1.3.0 +- package: github.com/rakyll/statik + version: 89fe3459b5c829c32e89bdff9c43f18aad728f2f + subpackages: + - fs - package: github.com/hashicorp/memberlist - package: golang.org/x/sync - package: github.com/CAFxX/gcnotifier diff --git a/gossip/gossip.go b/gossip/gossip.go index c3ded776a..1b15522a7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gossip import ( @@ -24,12 +38,13 @@ type GossipNodeSet struct { broadcasts *memberlist.TransmitLimitedQueue statusHandler pilosa.StatusHandler - config *GossipConfig + config *gossipConfig // The writer for any logging. LogOutput io.Writer } +// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. func (g *GossipNodeSet) Nodes() []*pilosa.Node { a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { @@ -38,11 +53,13 @@ func (g *GossipNodeSet) Nodes() []*pilosa.Node { return a } +// Start implements the BroadcastReceiver interface and sets the BroadcastHandler func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil } +// Open implements the NodeSet interface to start network activity. func (g *GossipNodeSet) Open() error { if g.handler == nil { return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") @@ -75,7 +92,7 @@ func (g *GossipNodeSet) logger() *log.Logger { //////////////////////////////////////////////////////////////// -type GossipConfig struct { +type gossipConfig struct { gossipSeed string memberlistConfig *memberlist.Config } @@ -87,7 +104,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed } //TODO: pull memberlist config from pilosa.cfg file - g.config = &GossipConfig{ + g.config = &gossipConfig{ memberlistConfig: memberlist.DefaultLocalConfig(), gossipSeed: gossipSeed, } @@ -103,7 +120,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed return g } -// SendSync implementation of the Broadcaster interface +// SendSync implementation of the Broadcaster interface. func (g *GossipNodeSet) SendSync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { @@ -131,7 +148,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { return eg.Wait() } -// SendAsync implementation of the Broadcaster interface +// SendAsync implementation of the Broadcaster interface. func (g *GossipNodeSet) SendAsync(pb proto.Message) error { msg, err := pilosa.MarshalMessage(pb) if err != nil { @@ -146,11 +163,13 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { return nil } -// implementation of the memberlist.Delegate interface +// NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} } +// NotifyMsg implementation of the memberlist.Delegate interface +// called when a user-data message is received. func (g *GossipNodeSet) NotifyMsg(b []byte) { m, err := pilosa.UnmarshalMessage(b) if err != nil { @@ -163,10 +182,14 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { } } +// GetBroadcasts implementation of the memberlist.Delegate interface +// called when user data messages can be broadcast. func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } +// LocalState implementation of the memberlist.Delegate interface +// sends this Node's state data. func (g *GossipNodeSet) LocalState(join bool) []byte { pb, err := g.statusHandler.LocalStatus() if err != nil { @@ -183,6 +206,8 @@ func (g *GossipNodeSet) LocalState(join bool) []byte { return buf } +// MergeRemoteState implementation of the memberlist.Delegate interface +// receive and process the remote side side's LocalState. func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeStatus diff --git a/handler.go b/handler.go index 1d2faf89f..3f7bb8845 100644 --- a/handler.go +++ b/handler.go @@ -1,3 +1,19 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate statik -src=./webui + package pilosa import ( @@ -23,6 +39,9 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + + _ "github.com/pilosa/pilosa/statik" + "github.com/rakyll/statik/fs" ) // Handler represents an HTTP handler. @@ -58,8 +77,11 @@ func NewHandler() *Handler { return handler } +// NewRouter creates a Gorilla Mux http router. func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() + router.HandleFunc("/", handler.handleWebUI).Methods("GET") + router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") @@ -83,7 +105,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST") router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/nodes", handler.handleGetNodes).Methods("GET") + router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") @@ -107,6 +129,21 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Router.ServeHTTP(w, r) } +func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { + // If user is using curl, don't chuck HTML at them + if strings.HasPrefix(r.UserAgent(), "curl") { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) + return + } + statikFS, err := fs.New() + if err != nil { + h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + fmt.Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + return + } + http.FileServer(statikFS).ServeHTTP(w, r) +} + // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSchemaResponse{ @@ -1237,8 +1274,8 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } } -// handleGetNodes handles /nodes requests. -func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) { +// handleGetHosts handles /hosts requests. +func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { h.logger().Printf("write version response error: %s", err) } @@ -1324,6 +1361,7 @@ type QueryResponse struct { Err error } +// MarshalJSON marshals QueryResponse into a JSON-encoded byte slice func (resp *QueryResponse) MarshalJSON() ([]byte, error) { var output struct { Results []interface{} `json:"results,omitempty"` diff --git a/handler_internal_test.go b/handler_internal_test.go index f749c23aa..169267c38 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/handler_test.go b/handler_test.go index d0b467f90..c2aa50927 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/holder.go b/holder.go index 8e71ebe5b..59fef1560 100644 --- a/holder.go +++ b/holder.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -46,7 +60,8 @@ func NewHolder() *Holder { indexes: make(map[string]*Index), closing: make(chan struct{}, 0), - Stats: NopStatsClient, + Broadcaster: NopBroadcaster, + Stats: NopStatsClient, CacheFlushInterval: DefaultCacheFlushInterval, @@ -350,7 +365,7 @@ type HolderSyncer struct { Closing <-chan struct{} } -// Returns true if the syncer has been marked to close. +// IsClosing returns true if the syncer has been marked to close. func (s *HolderSyncer) IsClosing() bool { select { case <-s.Closing: diff --git a/holder_test.go b/holder_test.go index 78c4700f8..d7c90fccd 100644 --- a/holder_test.go +++ b/holder_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/httpbroadcast/messenger.go b/httpbroadcast/messenger.go index ebd4b2e6a..387f28585 100644 --- a/httpbroadcast/messenger.go +++ b/httpbroadcast/messenger.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package httpbroadcast import ( @@ -62,11 +76,11 @@ func (h *HTTPBroadcaster) SendAsync(pb proto.Message) error { func (h *HTTPBroadcaster) nodes() ([]*pilosa.Node, error) { if h.server == nil { - return nil, errors.New("HTTPBroadcaster has no reference to Server.") + return nil, errors.New("HTTPBroadcaster has no reference to Server") } nodeset, ok := h.server.Cluster.NodeSet.(*HTTPNodeSet) if !ok { - return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet.") + return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet") } return nodeset.Nodes(), nil } @@ -106,12 +120,14 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { return nil } +// HTTPBroadcastReceiver unmarshals incoming messages over HTTP and passes them on to the handler. type HTTPBroadcastReceiver struct { port string handler pilosa.BroadcastHandler logOutput io.Writer } +// NewHTTPBroadcastReceiver returns a new instance of HTTPBroadcastReceiver. func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { return &HTTPBroadcastReceiver{ port: port, @@ -119,6 +135,7 @@ func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastRe } } +// Start implements the BroadcastReceiver interface and starts listening for broadcast messages. func (rec *HTTPBroadcastReceiver) Start(b pilosa.BroadcastHandler) error { rec.handler = b go func() { @@ -166,14 +183,17 @@ func NewHTTPNodeSet() *HTTPNodeSet { return &HTTPNodeSet{} } +// Nodes implements the NodeSet interface and returns a list of nodes in the cluster. func (h *HTTPNodeSet) Nodes() []*pilosa.Node { return h.nodes } +// Open implements the NodeSet interface to start network activity, but for a HTTPNodeSet it does nothing. func (h *HTTPNodeSet) Open() error { return nil } +// Join sets the NodeSet nodes to the slice of Nodes passed in. func (h *HTTPNodeSet) Join(nodes []*pilosa.Node) error { h.nodes = nodes return nil diff --git a/index.go b/index.go index 516064f64..e400b19ce 100644 --- a/index.go +++ b/index.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -68,8 +82,9 @@ func NewIndex(path, name string) (*Index, error) { columnLabel: DefaultColumnLabel, - Stats: NopStatsClient, - LogOutput: ioutil.Discard, + broadcaster: NopBroadcaster, + Stats: NopStatsClient, + LogOutput: ioutil.Discard, }, nil } @@ -93,7 +108,7 @@ func (i *Index) SetColumnLabel(v string) error { } // Make sure columnLabel is valid name - err := ValidateName(v) + err := ValidateLabel(v) if err != nil { return err } @@ -248,6 +263,7 @@ func (i *Index) MaxSlice() uint64 { return max } +// SetRemoteMaxSlice sets the remote max slice value received from another node. func (i *Index) SetRemoteMaxSlice(newmax uint64) { i.mu.Lock() defer i.mu.Unlock() @@ -271,6 +287,7 @@ func (i *Index) MaxInverseSlice() uint64 { return max } +// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node. func (i *Index) SetRemoteMaxInverseSlice(v uint64) { i.mu.Lock() defer i.mu.Unlock() @@ -375,7 +392,11 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } // Default the time quantum to what is set on the Index. - if err := f.SetTimeQuantum(i.timeQuantum); err != nil { + timeQuantum := i.timeQuantum + if opt.TimeQuantum != "" { + timeQuantum = opt.TimeQuantum + } + if err := f.SetTimeQuantum(timeQuantum); err != nil { f.Close() return nil, err } @@ -519,7 +540,7 @@ func encodeIndex(d *Index) *internal.Index { ColumnLabel: d.columnLabel, TimeQuantum: string(d.timeQuantum), }, - MaxSlice: d.remoteMaxSlice, + MaxSlice: d.MaxSlice(), Frames: encodeFrames(d.Frames()), } } diff --git a/index_test.go b/index_test.go index 67b5b8b2c..7986855e6 100644 --- a/index_test.go +++ b/index_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -34,23 +48,43 @@ func TestIndex_CreateFrameIfNotExists(t *testing.T) { } } -// Ensure index defaults the time quantum on new frames. +// Ensure index is assigned the correct time quantum on creation. func TestIndex_CreateFrame_TimeQuantum(t *testing.T) { - index := MustOpenIndex() - defer index.Close() + t.Run("Explicit", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } - // Create frame. - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { - t.Fatalf("unexpected frame time quantum: %s", q) - } + // Create frame with explicit quantum. + f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) + if err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected frame time quantum: %s", q) + } + }) + + t.Run("Inherited", func(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } + + // Create frame. + f, err := index.CreateFrame("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { + t.Fatalf("unexpected frame time quantum: %s", q) + } + }) } // Ensure index can delete a frame. diff --git a/internal/internal.go b/internal/internal.go index d727d97e9..e316aba1b 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package internal import ( diff --git a/internal/private.pb.go b/internal/private.pb.go index a41df4788..c5d7be01e 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-gogo. // source: private.proto // DO NOT EDIT! @@ -126,8 +140,9 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { } type CreateSliceMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` + IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"` } func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } @@ -211,6 +226,7 @@ type Index struct { Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"` } func (m *Index) Reset() { *m = Index{} } @@ -582,6 +598,16 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) } + if m.IsInverse { + dAtA[i] = 0x18 + i++ + if m.IsInverse { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -795,6 +821,23 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.Slices) > 0 { + dAtA12 := make([]byte, len(m.Slices)*10) + var j11 int + for _, num := range m.Slices { + for num >= 1<<7 { + dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j11++ + } + dAtA12[j11] = uint8(num) + j11++ + } + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(j11)) + i += copy(dAtA[i:], dAtA12[:j11]) + } return i, nil } @@ -1026,6 +1069,9 @@ func (m *CreateSliceMessage) Size() (n int) { if m.Slice != 0 { n += 1 + sovPrivate(uint64(m.Slice)) } + if m.IsInverse { + n += 2 + } return n } @@ -1119,6 +1165,13 @@ func (m *Index) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if len(m.Slices) > 0 { + l = 0 + for _, e := range m.Slices { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } return n } @@ -2224,6 +2277,26 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsInverse = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2938,6 +3011,68 @@ func (m *Index) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3287,44 +3422,45 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 617 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40, - 0x10, 0xc5, 0x89, 0x53, 0x9a, 0xa9, 0x5a, 0xda, 0xa5, 0x42, 0xa6, 0x42, 0x51, 0xb4, 0x07, 0x5a, - 0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, 0xdf, - 0x24, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0xdc, 0xf9, 0x03, 0x24, 0xbe, 0x81, - 0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0xc7, 0x6b, 0x3b, 0xb8, 0x94, 0x0a, 0x6e, 0x3b, - 0x6f, 0x66, 0xe7, 0xbd, 0x7d, 0x9e, 0x31, 0x6c, 0x4e, 0x63, 0x7f, 0x26, 0x35, 0x1e, 0x4d, 0xe3, - 0x48, 0x47, 0x6c, 0xdd, 0x0f, 0x35, 0xc6, 0xa1, 0x0c, 0xf8, 0x2b, 0xa8, 0x77, 0xc3, 0x11, 0x5e, - 0xf6, 0x50, 0x4b, 0xd6, 0x84, 0x8d, 0x56, 0x14, 0x24, 0x93, 0xf0, 0xa5, 0x1c, 0x60, 0xe0, 0x39, - 0x4d, 0xe7, 0xa0, 0x2e, 0x56, 0x21, 0x53, 0x71, 0xe1, 0x4f, 0xf0, 0x75, 0x22, 0x43, 0x9d, 0x4c, - 0xbc, 0x4a, 0x5a, 0xb1, 0x02, 0xf1, 0xaf, 0x0e, 0xd4, 0x5f, 0xc4, 0x72, 0x82, 0xd4, 0x71, 0x0f, - 0xd6, 0x45, 0x34, 0x5f, 0x6d, 0x97, 0xc7, 0xec, 0x21, 0x6c, 0x75, 0xc3, 0x19, 0xc6, 0x0a, 0x3b, - 0xa1, 0x1c, 0x04, 0x38, 0xa2, 0x76, 0xeb, 0xa2, 0x84, 0xb2, 0x07, 0x50, 0x6f, 0xc9, 0xe1, 0x3b, - 0xbc, 0x58, 0x4c, 0xd1, 0xab, 0x52, 0x93, 0x02, 0xc8, 0xb3, 0x7d, 0xff, 0x03, 0x7a, 0x6e, 0xd3, - 0x39, 0xd8, 0x14, 0x05, 0x50, 0xd6, 0x5b, 0xbb, 0xaa, 0x97, 0xc3, 0x56, 0x77, 0x32, 0x8d, 0x62, - 0x2d, 0x50, 0x4d, 0xa3, 0x50, 0x21, 0xdb, 0x86, 0x6a, 0x27, 0x8e, 0xad, 0x5c, 0x73, 0xe4, 0x1f, - 0x61, 0xfb, 0x34, 0x88, 0x86, 0xe3, 0xb6, 0xd4, 0x52, 0xe0, 0xfb, 0x04, 0x95, 0x66, 0xbb, 0x50, - 0x23, 0xe3, 0x6c, 0x5d, 0x1a, 0x18, 0x94, 0x1e, 0x6f, 0x9d, 0x49, 0x03, 0x83, 0xd2, 0x7d, 0x52, - 0xef, 0x8a, 0x34, 0x30, 0x68, 0x3f, 0xf0, 0x87, 0xa9, 0x6a, 0x57, 0xa4, 0x01, 0x63, 0xe0, 0xbe, - 0xf1, 0x71, 0x6e, 0xa5, 0xd2, 0x99, 0x77, 0x61, 0x67, 0x85, 0xdf, 0xca, 0xbc, 0x07, 0x6b, 0x22, - 0x9a, 0x77, 0xdb, 0xca, 0x73, 0x9a, 0xd5, 0x03, 0x57, 0xd8, 0x88, 0x0c, 0xa1, 0x2f, 0x66, 0x52, - 0x15, 0x4a, 0x15, 0x00, 0xbf, 0x0f, 0x35, 0x72, 0xc7, 0xbc, 0xb2, 0xb8, 0x6b, 0x8e, 0xfc, 0x8b, - 0x03, 0x3b, 0x3d, 0x79, 0x49, 0x32, 0x54, 0x4e, 0x73, 0x06, 0xf5, 0x1c, 0xa4, 0xea, 0x8d, 0xe3, - 0xc3, 0xa3, 0x6c, 0x7c, 0x8e, 0xae, 0xd4, 0x17, 0x48, 0x27, 0xd4, 0xf1, 0x42, 0x14, 0x97, 0xf7, - 0x9e, 0xc3, 0xd6, 0xef, 0x49, 0xa3, 0x61, 0x8c, 0x8b, 0xcc, 0xe9, 0x31, 0x2e, 0x8c, 0x27, 0x33, - 0x19, 0x24, 0xa9, 0x7f, 0xae, 0x48, 0x83, 0xa7, 0x95, 0x27, 0x0e, 0x3f, 0x01, 0xd6, 0x8a, 0x51, - 0x6a, 0xa4, 0x06, 0x3d, 0x54, 0x4a, 0xbe, 0xc5, 0xeb, 0xbf, 0x42, 0xea, 0x6c, 0x65, 0xc5, 0x59, - 0x7e, 0x08, 0xac, 0x8d, 0x01, 0x6a, 0xb4, 0x03, 0xff, 0x97, 0x0e, 0xbc, 0x9f, 0xb1, 0xdd, 0x5c, - 0xcb, 0xf6, 0xc1, 0x35, 0xb3, 0x4e, 0x64, 0x1b, 0xc7, 0x77, 0x0b, 0x73, 0xf2, 0xc5, 0x12, 0x54, - 0xc0, 0xfd, 0xac, 0xa9, 0xdd, 0x8f, 0x1b, 0x9e, 0xf0, 0x87, 0x41, 0xca, 0xa8, 0xaa, 0x65, 0xaa, - 0x7c, 0xe3, 0x2c, 0xd5, 0x49, 0xf6, 0xd6, 0xff, 0xa5, 0xe2, 0x6d, 0x8b, 0x9a, 0x81, 0x3c, 0x37, - 0xd9, 0xf4, 0x0e, 0x9d, 0xaf, 0x7f, 0x72, 0x59, 0xc7, 0x27, 0xc7, 0x52, 0xfe, 0x5b, 0x9b, 0x92, - 0x73, 0xe6, 0x37, 0x92, 0x8d, 0x8e, 0xdd, 0xa1, 0x3c, 0x66, 0xfb, 0xb0, 0x46, 0xac, 0xca, 0x73, - 0x69, 0x3a, 0xef, 0x94, 0xd4, 0x08, 0x9b, 0xe6, 0x12, 0xe0, 0x3c, 0x1a, 0x61, 0x5f, 0x4b, 0x9d, - 0x28, 0xa3, 0xe7, 0x2c, 0x52, 0x3a, 0xd3, 0x63, 0xce, 0x34, 0x37, 0x5a, 0xea, 0xdc, 0x09, 0x0a, - 0xd8, 0x23, 0xb8, 0x4d, 0x7a, 0x50, 0x79, 0xd5, 0x32, 0x03, 0x25, 0x44, 0x96, 0xe7, 0xcf, 0x60, - 0xb3, 0x15, 0x24, 0x4a, 0x63, 0x6c, 0x59, 0x0e, 0xa1, 0x66, 0x38, 0xb3, 0xcd, 0xd9, 0x2d, 0x6e, - 0x16, 0x52, 0x44, 0x5a, 0x72, 0xba, 0xfd, 0x6d, 0xd9, 0x70, 0xbe, 0x2f, 0x1b, 0xce, 0x8f, 0x65, - 0xc3, 0xf9, 0xfc, 0xb3, 0x71, 0x6b, 0xb0, 0x46, 0x7f, 0xeb, 0xc7, 0xbf, 0x02, 0x00, 0x00, 0xff, - 0xff, 0x77, 0x02, 0x29, 0xbb, 0xbe, 0x05, 0x00, 0x00, + // 640 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x4e, 0x14, 0x4d, + 0x10, 0xfe, 0x67, 0x77, 0x96, 0x7f, 0xa7, 0x08, 0x08, 0x2d, 0x31, 0x23, 0x21, 0x9b, 0x4d, 0x1f, + 0x04, 0x39, 0x70, 0xc0, 0x8b, 0x51, 0x0f, 0x86, 0x5d, 0x0c, 0x93, 0x08, 0xc6, 0x5e, 0xe2, 0xd1, + 0xa4, 0x81, 0x8a, 0x4e, 0x98, 0x9d, 0x59, 0xa7, 0x7b, 0x80, 0xf5, 0xe0, 0x73, 0x98, 0x78, 0xf2, + 0x01, 0x7c, 0x0f, 0x8f, 0x3e, 0x82, 0xc1, 0x17, 0x31, 0x5d, 0xdd, 0x33, 0xb3, 0x0e, 0x22, 0xd1, + 0x5b, 0xd7, 0x57, 0xd5, 0xf5, 0x7d, 0xfd, 0x4d, 0xd5, 0xc0, 0xc2, 0x24, 0x8f, 0xcf, 0xa4, 0xc6, + 0xad, 0x49, 0x9e, 0xe9, 0x8c, 0x75, 0xe3, 0x54, 0x63, 0x9e, 0xca, 0x84, 0xbf, 0x80, 0x20, 0x4a, + 0x4f, 0xf0, 0x62, 0x1f, 0xb5, 0x64, 0x7d, 0x98, 0x1f, 0x64, 0x49, 0x31, 0x4e, 0x9f, 0xcb, 0x23, + 0x4c, 0x42, 0xaf, 0xef, 0x6d, 0x04, 0x62, 0x16, 0x32, 0x15, 0x87, 0xf1, 0x18, 0x5f, 0x16, 0x32, + 0xd5, 0xc5, 0x38, 0x6c, 0xd9, 0x8a, 0x19, 0x88, 0x7f, 0xf1, 0x20, 0x78, 0x96, 0xcb, 0x31, 0x52, + 0xc7, 0x55, 0xe8, 0x8a, 0xec, 0x7c, 0xb6, 0x5d, 0x15, 0xb3, 0x7b, 0xb0, 0x18, 0xa5, 0x67, 0x98, + 0x2b, 0xdc, 0x4d, 0xe5, 0x51, 0x82, 0x27, 0xd4, 0xae, 0x2b, 0x1a, 0x28, 0x5b, 0x83, 0x60, 0x20, + 0x8f, 0xdf, 0xe2, 0xe1, 0x74, 0x82, 0x61, 0x9b, 0x9a, 0xd4, 0x40, 0x95, 0x1d, 0xc5, 0xef, 0x31, + 0xf4, 0xfb, 0xde, 0xc6, 0x82, 0xa8, 0x81, 0xa6, 0xde, 0xce, 0x55, 0xbd, 0x1c, 0x16, 0xa3, 0xf1, + 0x24, 0xcb, 0xb5, 0x40, 0x35, 0xc9, 0x52, 0x85, 0x6c, 0x09, 0xda, 0xbb, 0x79, 0xee, 0xe4, 0x9a, + 0x23, 0xff, 0x00, 0x4b, 0x3b, 0x49, 0x76, 0x7c, 0x3a, 0x94, 0x5a, 0x0a, 0x7c, 0x57, 0xa0, 0xd2, + 0x6c, 0x05, 0x3a, 0x64, 0x9c, 0xab, 0xb3, 0x81, 0x41, 0xe9, 0xf1, 0xce, 0x19, 0x1b, 0x18, 0x94, + 0xee, 0x93, 0x7a, 0x5f, 0xd8, 0xc0, 0xa0, 0xa3, 0x24, 0x3e, 0xb6, 0xaa, 0x7d, 0x61, 0x03, 0xc6, + 0xc0, 0x7f, 0x15, 0xe3, 0xb9, 0x93, 0x4a, 0x67, 0x1e, 0xc1, 0xf2, 0x0c, 0xbf, 0x93, 0x79, 0x07, + 0xe6, 0x44, 0x76, 0x1e, 0x0d, 0x55, 0xe8, 0xf5, 0xdb, 0x1b, 0xbe, 0x70, 0x11, 0x19, 0x42, 0x5f, + 0xcc, 0xa4, 0x5a, 0x94, 0xaa, 0x01, 0x7e, 0x17, 0x3a, 0xe4, 0x8e, 0x79, 0x65, 0x7d, 0xd7, 0x1c, + 0xf9, 0x27, 0x0f, 0x96, 0xf7, 0xe5, 0x05, 0xc9, 0x50, 0x15, 0xcd, 0x1e, 0x04, 0x15, 0x48, 0xd5, + 0xf3, 0xdb, 0x9b, 0x5b, 0xe5, 0xf8, 0x6c, 0x5d, 0xa9, 0xaf, 0x91, 0xdd, 0x54, 0xe7, 0x53, 0x51, + 0x5f, 0x5e, 0x7d, 0x02, 0x8b, 0xbf, 0x26, 0x8d, 0x86, 0x53, 0x9c, 0x96, 0x4e, 0x9f, 0xe2, 0xd4, + 0x78, 0x72, 0x26, 0x93, 0xc2, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0x6a, 0x3d, 0xf4, 0xf8, 0x6b, 0x60, + 0x83, 0x1c, 0xa5, 0x46, 0x6a, 0xb0, 0x8f, 0x4a, 0xc9, 0x37, 0x78, 0xfd, 0x57, 0xb0, 0xce, 0xb6, + 0x66, 0x9d, 0x5d, 0x83, 0x20, 0x52, 0x6e, 0xb6, 0xe8, 0x4b, 0x74, 0x45, 0x0d, 0xf0, 0x4d, 0x60, + 0x43, 0x4c, 0x50, 0xa3, 0x5b, 0x87, 0x3f, 0xf4, 0xe7, 0xa3, 0x52, 0xcb, 0xcd, 0xb5, 0x6c, 0x1d, + 0x7c, 0xb3, 0x09, 0x24, 0x65, 0x7e, 0xfb, 0x76, 0x6d, 0x5d, 0xb5, 0x76, 0x82, 0x0a, 0x78, 0x5c, + 0x36, 0x75, 0xdb, 0x73, 0xc3, 0x03, 0x7f, 0x33, 0x66, 0x25, 0x55, 0xbb, 0x49, 0x55, 0xed, 0xa3, + 0xa3, 0x7a, 0x5a, 0xbe, 0xf5, 0x5f, 0xa9, 0xf8, 0xd0, 0xa1, 0x66, 0x5c, 0x0f, 0x4c, 0xd6, 0xde, + 0xa1, 0xf3, 0xf5, 0x4f, 0x6e, 0xea, 0xf8, 0xec, 0x39, 0xca, 0xbf, 0x6b, 0xd3, 0x70, 0xce, 0xfc, + 0x64, 0xca, 0xc1, 0x72, 0x1b, 0x56, 0xc5, 0x6c, 0x1d, 0xe6, 0x88, 0x55, 0x85, 0x3e, 0xcd, 0xee, + 0xad, 0x86, 0x1a, 0xe1, 0xd2, 0x66, 0x9d, 0xdc, 0x90, 0x77, 0xec, 0x3a, 0xd9, 0x88, 0x4b, 0x80, + 0x83, 0xec, 0x04, 0x47, 0x5a, 0xea, 0x42, 0x19, 0x9d, 0x7b, 0x99, 0xd2, 0xa5, 0x4e, 0x73, 0xa6, + 0x69, 0xd3, 0x52, 0x57, 0x0e, 0x51, 0xc0, 0xee, 0xc3, 0xff, 0xa4, 0x13, 0x55, 0xd8, 0x6e, 0x32, + 0x53, 0x42, 0x94, 0x79, 0xfe, 0x18, 0x16, 0x06, 0x49, 0xa1, 0x34, 0xe6, 0x8e, 0x65, 0x13, 0x3a, + 0x86, 0xb3, 0xdc, 0xb7, 0x95, 0xfa, 0x66, 0x2d, 0x45, 0xd8, 0x92, 0x9d, 0xa5, 0xaf, 0x97, 0x3d, + 0xef, 0xdb, 0x65, 0xcf, 0xfb, 0x7e, 0xd9, 0xf3, 0x3e, 0xfe, 0xe8, 0xfd, 0x77, 0x34, 0x47, 0xff, + 0xf8, 0x07, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1c, 0x79, 0x08, 0xf4, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index b83683d51..c7c792f4a 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -43,6 +43,7 @@ message MaxSlicesResponse { message CreateSliceMessage { string Index = 1; uint64 Slice = 2; + bool IsInverse = 3; } message DeleteIndexMessage { @@ -75,6 +76,7 @@ message Index { IndexMeta Meta = 2; uint64 MaxSlice = 3; repeated Frame Frames = 4; + repeated uint64 Slices = 5; } message NodeStatus { diff --git a/internal/public.pb.go b/internal/public.pb.go index 9eb005c17..8499042fd 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-gogo. // source: public.proto // DO NOT EDIT! @@ -2576,7 +2590,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ // 576 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4, 0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b, diff --git a/iterator.go b/iterator.go index 155914491..473e862dd 100644 --- a/iterator.go +++ b/iterator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/iterator_test.go b/iterator_test.go index c9240b6c5..025672f8a 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/pilosa.go b/pilosa.go index 16739fd7a..dd7402677 100644 --- a/pilosa.go +++ b/pilosa.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -24,16 +38,19 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") + ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") ) -// Regular expression to valuate index and frame's name -// Todo: remove . when frame doesn't require . for topN -var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`) +// Regular expression to validate index and frame names. +var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,64}$`) + +// Regular expression to validate row and column labels. +var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,64}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. @@ -88,11 +105,18 @@ func decodeColumnAttrSet(pb *internal.ColumnAttrSet) *ColumnAttrSet { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// Restrict name using regex +// ValidateName ensures that the name is a valid format. func ValidateName(name string) error { - validName := nameRegexp.Match([]byte(name)) - if validName == false { + if nameRegexp.Match([]byte(name)) == false { return ErrName } return nil } + +// ValidateLabel ensures that the label is a valid format. +func ValidateLabel(label string) error { + if labelRegexp.Match([]byte(label)) == false { + return ErrLabel + } + return nil +} diff --git a/pql/ast.go b/pql/ast.go index 6673bb638..b83b35731 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( @@ -156,6 +170,30 @@ func (c *Call) String() string { return buf.String() } +// SupportsInverse indicates that the call may be on an inverse frame. +func (c *Call) SupportsInverse() bool { + if c.Name == "Bitmap" { + return true + } + return false +} + +// IsInverse specifies if the call is for an inverse view. +// Return defaults to false unless absolutely sure of inversion. +func (c *Call) IsInverse(rowLabel, columnLabel string) bool { + if c.SupportsInverse() { + _, rowOK, rowErr := c.UintArg(rowLabel) + _, columnOK, columnErr := c.UintArg(columnLabel) + if rowErr != nil || columnErr != nil { + return false + } + if !rowOK && columnOK { + return true + } + } + return false +} + // CopyArgs returns a copy of m. func CopyArgs(m map[string]interface{}) map[string]interface{} { other := make(map[string]interface{}, len(m)) diff --git a/pql/ast_test.go b/pql/ast_test.go index 653bd5af5..2b8519571 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql_test import ( @@ -15,3 +29,69 @@ func TestCall_String(t *testing.T) { } }) } + +// Ensure call can be converted into a string. +func TestCall_SupportsInverse(t *testing.T) { + t.Run("Bitmap", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap()`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].SupportsInverse() != true { + t.Fatalf("call should support inverse: %s", q.Calls[0]) + } + }) + t.Run("Count Bitmap", func(t *testing.T) { + q, err := pql.ParseString(`Count(Bitmap())`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].SupportsInverse() == true { + t.Fatalf("call should not support inverse: %s", q.Calls[0]) + } + }) + t.Run("Union Bitmaps", func(t *testing.T) { + q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].SupportsInverse() == true { + t.Fatalf("call should not support inverse: %s", q.Calls[0]) + } + }) + +} + +// Ensure call is correctly determined to be against an inverse view. +func TestCall_IsInverse(t *testing.T) { + t.Run("Bitmap Row", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap(frame="f", row=1)`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].IsInverse("row", "col") != false { + t.Fatalf("incorrect call inverse: %s", q.Calls[0]) + } + }) + t.Run("Bitmap Column", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].IsInverse("row", "col") != true { + t.Fatalf("incorrect call inverse: %s", q.Calls[0]) + } + }) + t.Run("Bitmap Column No Label", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].IsInverse("rowX", "colX") != false { + t.Fatalf("incorrect call inverse: %s", q.Calls[0]) + } + }) + t.Run("Count", func(t *testing.T) { + q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`) + if err != nil { + t.Fatal(err) + } else if q.Calls[0].IsInverse("row", "col") != false { + t.Fatalf("incorrect call inverse: %s", q.Calls[0]) + } + }) + +} diff --git a/pql/doc.go b/pql/doc.go index abd8607a1..6150f4012 100644 --- a/pql/doc.go +++ b/pql/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Package pql defines the Pilosa Query Language. */ diff --git a/pql/parser.go b/pql/parser.go index f7d93ab81..048f6a664 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( diff --git a/pql/parser_test.go b/pql/parser_test.go index 66b84f6be..f757a7d2e 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql_test import ( diff --git a/pql/scanner.go b/pql/scanner.go index a0146725a..a559d8ccb 100644 --- a/pql/scanner.go +++ b/pql/scanner.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( diff --git a/pql/scanner_test.go b/pql/scanner_test.go index 4dc2f4d8f..5b7a06b2f 100644 --- a/pql/scanner_test.go +++ b/pql/scanner_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql_test import ( diff --git a/pql/token.go b/pql/token.go index 2ffc84400..4d71117af 100644 --- a/pql/token.go +++ b/pql/token.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import "strings" diff --git a/roaring/assembly.go b/roaring/assembly.go index 3ffa6efc7..1e5d49519 100644 --- a/roaring/assembly.go +++ b/roaring/assembly.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring func hasAsm() bool diff --git a/roaring/assembly_asm.go b/roaring/assembly_asm.go index 88435422d..eec8234b8 100644 --- a/roaring/assembly_asm.go +++ b/roaring/assembly_asm.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build amd64 package roaring diff --git a/roaring/assembly_generic.go b/roaring/assembly_generic.go index 50c4936d5..746970ba0 100644 --- a/roaring/assembly_generic.go +++ b/roaring/assembly_generic.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !amd64 package roaring diff --git a/roaring/assembly_test.go b/roaring/assembly_test.go index 244e5ad16..f18aa65ec 100644 --- a/roaring/assembly_test.go +++ b/roaring/assembly_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import "testing" diff --git a/roaring/internal_test.go b/roaring/internal_test.go index 4a39906d3..59fbccf1c 100644 --- a/roaring/internal_test.go +++ b/roaring/internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring import ( diff --git a/roaring/roaring.go b/roaring/roaring.go index e74739311..e0810a6c2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // package roaring implements roaring bitmaps with support for incremental changes. package roaring diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index ccbe3b66c..d28e72354 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package roaring_test import ( @@ -197,7 +211,10 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { } // Verify slices are equal. - if got, exp := bm.Slice(), uint64SetSlice(m); !reflect.DeepEqual(got, exp) { + // If `got` is nil and `exp` has zero length, don't perform the DeepEqual + // because when `a` is empty (`a = []uint64{}`) then `got` is a nil slice + // while `exp` is an empty slice. Therefore they will not be considered equal. + if got, exp := bm.Slice(), uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp) } diff --git a/server.go b/server.go index 9ccb0ee84..60570bc0f 100644 --- a/server.go +++ b/server.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -254,9 +268,16 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - idx.SetRemoteMaxSlice(obj.Slice) + if obj.IsInverse { + idx.SetRemoteMaxInverseSlice(obj.Slice) + } else { + idx.SetRemoteMaxSlice(obj.Slice) + } case *internal.CreateIndexMessage: - opt := IndexOptions{ColumnLabel: obj.Meta.ColumnLabel} + opt := IndexOptions{ + ColumnLabel: obj.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), + } _, err := s.Holder.CreateIndex(obj.Index, opt) if err != nil { return err @@ -267,7 +288,13 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { } case *internal.CreateFrameMessage: index := s.Holder.Index(obj.Index) - opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + opt := FrameOptions{ + RowLabel: obj.Meta.RowLabel, + InverseEnabled: obj.Meta.InverseEnabled, + CacheType: obj.Meta.CacheType, + CacheSize: obj.Meta.CacheSize, + TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), + } _, err := index.CreateFrame(obj.Frame, opt) if err != nil { return err @@ -281,19 +308,27 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return nil } -// Server implements StatusHandler. // LocalStatus returns the state of the local node as well as the // holder (indexes/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. +// Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { if s.Holder == nil { return nil, errors.New("Server.Holder is nil") } - return &internal.NodeStatus{ + + ns := internal.NodeStatus{ Host: s.Host, State: NodeStateUp, Indexes: encodeIndexes(s.Holder.Indexes()), - }, nil + } + + // Append Slice list per this Node's indexes + for _, index := range ns.Indexes { + index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Host) + } + + return &ns, nil } // ClusterStatus returns the NodeState for all nodes in the cluster. @@ -308,6 +343,14 @@ func (s *Server) ClusterStatus() (proto.Message, error) { // Update NodeState for all nodes. for host, nodeState := range s.Cluster.NodeStates() { + // In a default configuration (or single-node) where a StaticNodeSet is used + // then all nodes are marked as DOWN. At the very least, we should consider + // the local node as UP. + // TODO: we should be able to remove this check if/when cluster.Nodes and + // cluster.NodeSet are unified. + if host == s.Host { + nodeState = NodeStateUp + } node := s.Cluster.NodeByHost(host) node.SetState(nodeState) } diff --git a/server/server.go b/server/server.go index 1f52c0f48..e2360522d 100644 --- a/server/server.go +++ b/server/server.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package server contains the `pilosa server` subcommand which runs Pilosa // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the diff --git a/server/server_test.go b/server/server_test.go index 4e3fa5e36..d981a4cf3 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server_test import ( @@ -8,16 +22,19 @@ import ( "io" "io/ioutil" "math/rand" + "net" "net/http" "os" "reflect" "sort" + "strconv" "strings" "testing" "testing/quick" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/httpbroadcast" "github.com/pilosa/pilosa/server" ) @@ -45,7 +62,7 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(rowID=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -61,7 +78,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -84,7 +101,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(rowID=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -111,7 +128,7 @@ func TestMain_SetRowAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil { t.Fatal(err) @@ -120,36 +137,36 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Set bits on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(rowID=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(rowID=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(rowID=1, frame="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(rowID=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } - // Query row x.n/1. - if res, err := m.Query("i", "", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row x/1. + if res, err := m.Query("i", "", `Bitmap(rowID=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } - // Query row x.n/2. - if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil { + // Query row x/2. + if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -160,19 +177,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - // Query row x.n/2. - if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil { + // Query row x/2. + if res, err := m.Query("i", "", `Bitmap(rowID=2, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -188,14 +205,14 @@ func TestMain_SetColumnAttrs(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", columnID=101)`); err != nil { t.Fatal(err) } @@ -205,7 +222,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -216,7 +233,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -232,14 +249,14 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set bits on row. - if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(rowID=1, frame="x", col=101)`); err != nil { t.Fatal(err) } @@ -249,7 +266,7 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(rowID=1, frame="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -282,19 +299,19 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m0.Query("x", "", ` - SetBit(id=1, frame="f", columnID=100) - SetBit(id=1, frame="f", columnID=1000) - SetBit(id=1, frame="f", columnID=100000) - SetBit(id=1, frame="f", columnID=200000) - SetBit(id=1, frame="f", columnID=400000) - SetBit(id=1, frame="f", columnID=600000) - SetBit(id=1, frame="f", columnID=800000) + SetBit(rowID=1, frame="f", columnID=100) + SetBit(rowID=1, frame="f", columnID=1000) + SetBit(rowID=1, frame="f", columnID=100000) + SetBit(rowID=1, frame="f", columnID=200000) + SetBit(rowID=1, frame="f", columnID=400000) + SetBit(rowID=1, frame="f", columnID=600000) + SetBit(rowID=1, frame="f", columnID=800000) `); err != nil { t.Fatal(err) } // Query row on first cluster. - if res, err := m0.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil { + if res, err := m0.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -317,7 +334,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on second cluster. - if res, err := m2.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil { + if res, err := m2.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -354,6 +371,160 @@ path = "/path/to/plugins" } } +// Ensure program can send/receive broadcast messages. +func TestMain_SendReceiveMessage(t *testing.T) { + m0 := MustRunMain() + defer m0.Close() + + m1 := MustRunMain() + defer m1.Close() + + // Get available ports for internal messaging + freePorts, err := availablePorts(2) + if err != nil { + t.Fatal(err) + } + + // Update cluster config + m0.Server.Cluster.Nodes = []*pilosa.Node{ + {Host: m0.Server.Host, InternalHost: "localhost:" + freePorts[0]}, + {Host: m1.Server.Host, InternalHost: "localhost:" + freePorts[1]}, + } + m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes + + // Configure node0 + m0.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m0.Server, freePorts[0]) + m0.Server.Handler.Broadcaster = m0.Server.Broadcaster + m0.Server.Holder.Broadcaster = m0.Server.Broadcaster + m0.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(freePorts[0], nil) + m0.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() + err = m0.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m0.Server.Cluster.Nodes) + if err != nil { + t.Fatal(err) + } + if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { + t.Fatal(err) + } + + // Configure node1 + m1.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m1.Server, freePorts[1]) + m1.Server.Handler.Broadcaster = m1.Server.Broadcaster + m1.Server.Holder.Broadcaster = m1.Server.Broadcaster + m1.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(freePorts[1], nil) + m1.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() + err = m1.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m1.Server.Cluster.Nodes) + if err != nil { + t.Fatal(err) + } + if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { + t.Fatal(err) + } + + // Expected indexes and Frames + expected := map[string][]string{ + "i": []string{"f"}, + } + + // Create a client for each node. + client0 := m0.Client() + client1 := m1.Client() + + // Create indexes and frames on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Make sure node0 knows about the index and frame created. + schema0, err := client0.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received0 := map[string][]string{} + for _, idx := range schema0 { + received0[idx.Name] = []string{} + for _, frame := range idx.Frames { + received0[idx.Name] = append(received0[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received0, expected) { + t.Fatalf("unexpected schema on node0: %d", received0) + } + + // Make sure node1 knows about the index and frame created. + schema1, err := client1.Schema(context.Background()) + if err != nil { + t.Fatal(err) + } + received1 := map[string][]string{} + for _, idx := range schema1 { + received1[idx.Name] = []string{} + for _, frame := range idx.Frames { + received1[idx.Name] = append(received1[idx.Name], frame.Name) + } + } + if !reflect.DeepEqual(received1, expected) { + t.Fatalf("unexpected schema on node1: %d", received1) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + SetBit(rowID=1, frame="f", columnID=1) + SetBit(rowID=1, frame="f", columnID=2400000) + `); err != nil { + t.Fatal(err) + } + + // Make sure node0 knows about the latest MaxSlice. + maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices0["i"] != 2 { + t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + } + + // Make sure node1 knows about the latest MaxSlice. + maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + if err != nil { + t.Fatal(err) + } + if maxSlices1["i"] != 2 { + t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + } +} + +// availablePorts returns a slice of ports that can be used for testing. +func availablePorts(cnt int) ([]string, error) { + rtn := []string{} + + for i := 0; i < cnt; i++ { + port, err := getPort() + if err != nil { + return nil, err + } + rtn = append(rtn, strconv.Itoa(port)) + } + return rtn, nil +} + +// Ask the kernel for a free open port that is ready to use +func getPort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + + return l.Addr().(*net.TCPAddr).Port, nil +} + // Main represents a test wrapper for main.Main. type Main struct { *server.Command @@ -432,7 +603,6 @@ func (m *Main) Client() *pilosa.Client { // Query executes a query against the program through the HTTP API. func (m *Main) Query(index, rawQuery, query string) (string, error) { - fmt.Println("Query:", index, query) resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) @@ -482,7 +652,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { for i := range cmds { cmds[i] = SetCommand{ ID: uint64(rand.Intn(1000)), - Frame: "x.n", + Frame: "x", ColumnID: uint64(rand.Intn(10)), } } diff --git a/statik/.gitignore b/statik/.gitignore new file mode 100644 index 000000000..514ee40a1 --- /dev/null +++ b/statik/.gitignore @@ -0,0 +1 @@ +statik.go diff --git a/statik/doc.go b/statik/doc.go new file mode 100644 index 000000000..85edd9e6f --- /dev/null +++ b/statik/doc.go @@ -0,0 +1,3 @@ +// Package statik contains static assets for the Web UI. `go generate` will +// produce statik.go, which is ignored by git. +package statik diff --git a/stats.go b/stats.go index 057d46320..efaa200d5 100644 --- a/stats.go +++ b/stats.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -12,7 +26,7 @@ func init() { NopStatsClient = &nopStatsClient{} } -// Global expvar. +// Expvar global expvar map. var Expvar = expvar.NewMap("index") // StatsClient represents a client to a stats server. @@ -42,9 +56,9 @@ type StatsClient interface { Timing(name string, value time.Duration) } +// NopStatsClient represents a client that doesn't do anything. var NopStatsClient StatsClient -// nopStatsClient represents a client that doesn't do anything. type nopStatsClient struct{} func (c *nopStatsClient) Tags() []string { return nil } diff --git a/time.go b/time.go index b2176285b..ed6f62bf0 100644 --- a/time.go +++ b/time.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/time_test.go b/time_test.go index 6de3df364..0602b60de 100644 --- a/time_test.go +++ b/time_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/view.go b/view.go index 9f67b516d..3f26185e4 100644 --- a/view.go +++ b/view.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -9,6 +23,8 @@ import ( "strconv" "strings" "sync" + + "github.com/pilosa/pilosa/internal" ) // View layout modes. @@ -36,7 +52,12 @@ type View struct { cacheType string // passed in by frame fragments map[uint64]*Fragment - stats StatsClient + // maxSlice maintains this view's max slice in order to + // prevent sending multiple `CreateSliceMessage` messages + maxSlice uint64 + + broadcaster Broadcaster + stats StatsClient RowAttrStore *AttrStore LogOutput io.Writer @@ -54,8 +75,9 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View { cacheType: DefaultCacheType, fragments: make(map[uint64]*Fragment), - stats: NopStatsClient, - LogOutput: ioutil.Discard, + broadcaster: NopBroadcaster, + stats: NopStatsClient, + LogOutput: ioutil.Discard, } } @@ -207,6 +229,22 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } frag.RowAttrStore = v.RowAttrStore + // Broadcast a message that a new max slice was just created. + if slice > v.maxSlice { + v.maxSlice = slice + + // Send the create slice message to all nodes. + err := v.broadcaster.SendAsync( + &internal.CreateSliceMessage{ + Index: v.index, + Slice: slice, + IsInverse: IsInverseView(v.name), + }) + if err != nil { + return nil, err + } + } + // Save to lookup. v.fragments[slice] = frag diff --git a/view_test.go b/view_test.go index adaa0ced2..53310c298 100644 --- a/view_test.go +++ b/view_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/webui/assets/chevron-down.png b/webui/assets/chevron-down.png new file mode 100644 index 000000000..3312489a2 Binary files /dev/null and b/webui/assets/chevron-down.png differ diff --git a/webui/assets/main.js b/webui/assets/main.js new file mode 100644 index 000000000..4e17db19a --- /dev/null +++ b/webui/assets/main.js @@ -0,0 +1,390 @@ +class REPL { + constructor(input, output, button) { + this.input = input + this.output = output + this.button = button + this.history = [] + this.history_index = 0 + this.history_buffer = '' + this.result_number = 0 + } + bind_events() { + var repl = this + var keys = { + TAB: 9, + ENTER: 13, + UP_ARROW: 38, + DOWN_ARROW: 40 + } + var keywords = { + // keyword: length of substring that comes after cursor + "SetBit()": 1, + "ClearBit()": 1, + "SetBitmapAttrs()": 1, + "Bitmap()": 1, + "Union()": 1, + "Intersect()": 1, + "Difference()": 1, + "Count()": 1, + "Range()": 1, + "TopN()": 1, + "frame=": 0, + } + + this.input.addEventListener("keydown", function(e) { + if (e.keyCode == keys.UP_ARROW) { + e.preventDefault() + if (repl.input.value.substring(0, repl.input.selectionStart).indexOf('\n') == '-1') { + if (repl.history_index == 0) { + return + } else { + if (repl.history_index == repl.history.length) { + repl.history_buffer = repl.input.value + } + repl.history_index-- + repl.input.value = repl.history[repl.history_index] + repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) + } + } + } + if (e.keyCode == keys.DOWN_ARROW) { + e.preventDefault() + if (repl.input.value.substring(repl.input.selectionEnd, repl.input.length).indexOf('\n') == '-1') { + if (repl.history_index == repl.history.length) { + return + } else { + repl.history_index++ + if (repl.history_index == repl.history.length) { + repl.input.value = repl.history_buffer + } else { + repl.input.value = repl.history[repl.history_index] + } + repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) + } + } + } + if (e.keyCode == keys.ENTER && !e.shiftKey) { + e.preventDefault() + repl.submit(); + } + if (e.keyCode == keys.TAB) { + e.preventDefault() + + // extract word fragment ending at cursor. a word fragment: + // - starts with last nonalpha character before cursor (or beginning of string) + // - ends at cursor + var word_start = repl.input.selectionEnd-1 + while(word_start>0) { + var c = repl.input.value.charCodeAt(word_start) + if(!((c>64 && c<91) || (c>96 && c<123))) { + word_start++ + break + } + word_start-- + } + var input_word = repl.input.value.substring(word_start, repl.input.selectionEnd) + + // check for keyword match and insert + // this just stops at the first match + for(var keyword in keywords) { + if(keyword.startsWith(input_word)){ + var cursor_pos = repl.input.selectionEnd + var completion = keyword.substring(input_word.length) + var before = repl.input.value.substring(0, cursor_pos) + var after = repl.input.value.substring(cursor_pos) + repl.input.value = before + completion + after + var new_pos = cursor_pos + completion.length - keywords[keyword] + repl.input.setSelectionRange(new_pos, new_pos) + break + } + } + } + }) + repl.button.onclick = function() { + repl.submit(); + }; + } + + submit() { + this.history_buffer = '' + this.history_index = this.history.length + this.history[this.history_index] = this.input.value + this.history_index++ + this.process_query(this.input.value) + this.input.value = "" + } + + process_query(query) { + var xhr = new XMLHttpRequest(); + var e = document.getElementById("index-dropdown"); + var indexname = e.options[e.selectedIndex].text; + xhr.open('POST', '/index/' + indexname + '/query'); + xhr.setRequestHeader('Content-Type', 'application/text'); + + var repl = this + var start_time = new Date().getTime(); + xhr.send(query) + xhr.onload = function() { + var end_time = new Date().getTime() + repl.result_number++ + repl.createSingleOutput({ + "input": query, + "output": xhr.responseText, + "indexname": indexname, + "querytime_ms": end_time - start_time, + }) + } + + } + + createSingleOutput(res) { + var node = document.createElement("div"); + node.classList.add('output'); + var output_string = res['output'] + var output_json = JSON.parse(output_string) + var result_class = "result-output" + var getting_started_errors = [ + 'index not found', + 'frame not found', + ] + + if("error" in output_json) { + result_class = "result-error" + if(getting_started_errors.indexOf(output_json['error']) >= 0) { + output_string += `
+
+ Just getting started? Try this:
+ $ curl -XPOST "http://127.0.0.1:10101/index/test" -d '{"options": {"columnLabel": "col"}}' # create index "test"
+ $ curl -XPOST "http://127.0.0.1:10101/index/test/frame/foo" -d '{"options": {"rowLabel": "row"}}' # create frame "foo"
+ # Select "test" in the index dropdown above
+ SetBit(row=0, col=0, frame=foo) # Use PQL to set a bit + ` + } + } + + + var markup =` +
+
+
+
+
Input
+        + Source: ${res.indexname} +
+
+ ${res.input} +
+
+
+
+
output
+        + ${res.querytime_ms} ms +
+
+ ${output_string} +
+
+
+
+
+ + Export raw .csv +
+
+
+ ` + node.innerHTML = markup; + this.output.insertBefore(node, this.output.firstChild) + } + + populate_index_dropdown() { + var xhr = new XMLHttpRequest(); + xhr.open('GET', '/schema') + var select = document.getElementById('index-dropdown') + + xhr.onload = function() { + var schema = JSON.parse(xhr.responseText) + for(var i=0; i 0) { + select.value = 1; + } + } + xhr.send(null) + } + +} + +function populate_version() { + var xhr = new XMLHttpRequest(); + xhr.open('GET', '/version') + var node = document.getElementById('server-version') + + xhr.onload = function() { + version = JSON.parse(xhr.responseText)['version'] + node.innerHTML = version + } + xhr.send(null) +} + +function handle_nav_click(e) { + // e.id = "nav-xxx" + name = e.id.substring(4) + set_active_pane_by_name(name) + window.location.hash = name +} + +function set_active_pane_by_name(name) { + // toggle the nav buttons + document.getElementsByClassName("nav-active")[0].classList.remove("nav-active") + document.getElementById("nav-" + name).classList.add("nav-active") + + // toggle the main interface content divs + document.getElementsByClassName("interface-active")[0].classList.remove("interface-active") + document.getElementById('interface-' + name).classList.add("interface-active") + + // hack hack + switch(name) { + case "cluster": + update_cluster_status() + break + case "documentation": + open_external_docs() + break + } +} + + +function update_cluster_status() { + var xhr = new XMLHttpRequest(); + xhr.open('GET', '/status') + status_node = document.getElementById('status') + xhr.onload = function() { + var status = JSON.parse(xhr.responseText) + render_status(status) + } + xhr.send(null) +} + +function render_status(status) { + // render node table + var nodes_div = document.getElementById("status-nodes") + while (nodes_div.firstChild) { + nodes_div.removeChild(nodes_div.firstChild); + } + + var nodes = status["status"]["Nodes"] + table = document.createElement("table") + tbody = document.createElement("tbody") + table.appendChild(tbody) + var caption = document.createElement("caption") + caption.innerHTML = "(" + nodes.length + ")" + table.appendChild(caption) + + var header = document.createElement('tr') + markup = `Host + State` + header.innerHTML = markup + tbody.appendChild(header) + for(var n=0; n${nodes[n]["Host"]} + ${nodes[n]["State"]}` + row.innerHTML = markup + tbody.appendChild(row) + } + nodes_div.appendChild(table) + + // render index tables + var indexes_div = document.getElementById("status-indexes") + while (indexes_div.firstChild) { + indexes_div.removeChild(indexes_div.firstChild); + } + + var indexes = nodes[0]["Indexes"] // TODO currently comes from only node 0 + for(var n=0; nName + Row Label + Cache Type + Cache Size` + header.innerHTML = markup + tbody.appendChild(header) + + var frames = indexes[n]["Frames"] + if(frames) { + for(var m=0; m${frames[m]["Name"]} + ${frames[m]["Meta"]["RowLabel"]} + ${frames[m]["Meta"]["CacheType"]} + ${frames[m]["Meta"]["CacheSize"]}` + tbody.appendChild(row) + } + } + indexes_div.appendChild(table) + } + + // render slice tables + // TODO enable when Slices element is present in status response + /* + var slices_div = document.getElementById("status-slices") + data = "" + for(var n=0; n" + } + } + slices_div.innerHTML = data + */ + +} + +function open_external_docs() { + window.open("https://www.pilosa.com/docs"); +} + +function check_anchor_uri() { + var pane_names = {"console": 0, "cluster": 0, "documentation": 0} + var anchor = window.location.hash.substr(1); + if(anchor in pane_names) { + set_active_pane_by_name(anchor) + } +} + +Date.prototype.today = function () { + return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate(); +} + +Date.prototype.timeNow = function () { + return ((this.getHours() < 10)?"0":"") + this.getHours() +":"+ ((this.getMinutes() < 10)?"0":"") + this.getMinutes() +":"+ ((this.getSeconds() < 10)?"0":"") + this.getSeconds(); +} + +populate_version() + +var input = document.getElementById('query') +var output = document.getElementById('outputs') +var button = document.getElementById('query-btn') + +repl = new REPL(input, output, button) +repl.populate_index_dropdown() +repl.bind_events() + +input.focus() + +check_anchor_uri() diff --git a/webui/assets/nav-cluster-active.svg b/webui/assets/nav-cluster-active.svg new file mode 100644 index 000000000..871d6f2af --- /dev/null +++ b/webui/assets/nav-cluster-active.svg @@ -0,0 +1 @@ +nav_cluster_1 diff --git a/webui/assets/nav-cluster.svg b/webui/assets/nav-cluster.svg new file mode 100644 index 000000000..baf6310ea --- /dev/null +++ b/webui/assets/nav-cluster.svg @@ -0,0 +1 @@ +nav_cluster_1 \ No newline at end of file diff --git a/webui/assets/nav-console-active.svg b/webui/assets/nav-console-active.svg new file mode 100644 index 000000000..2263c2a69 --- /dev/null +++ b/webui/assets/nav-console-active.svg @@ -0,0 +1 @@ +nav_console \ No newline at end of file diff --git a/webui/assets/nav-console.svg b/webui/assets/nav-console.svg new file mode 100644 index 000000000..e9d2a9569 --- /dev/null +++ b/webui/assets/nav-console.svg @@ -0,0 +1 @@ +nav_console diff --git a/webui/assets/nav-documentation-active.svg b/webui/assets/nav-documentation-active.svg new file mode 100644 index 000000000..b6e994980 --- /dev/null +++ b/webui/assets/nav-documentation-active.svg @@ -0,0 +1 @@ +documentation diff --git a/webui/assets/nav-documentation.svg b/webui/assets/nav-documentation.svg new file mode 100644 index 000000000..12cd2108c --- /dev/null +++ b/webui/assets/nav-documentation.svg @@ -0,0 +1 @@ +documentation \ No newline at end of file diff --git a/webui/assets/nav_item1.svg b/webui/assets/nav_item1.svg new file mode 100644 index 000000000..d6d7f779e --- /dev/null +++ b/webui/assets/nav_item1.svg @@ -0,0 +1 @@ +nav_item1 \ No newline at end of file diff --git a/webui/assets/style.css b/webui/assets/style.css new file mode 100644 index 000000000..a3a493e16 --- /dev/null +++ b/webui/assets/style.css @@ -0,0 +1,293 @@ +*{ + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +body{ + font-family: sans-serif; + background-color: #fbfcfd; + margin: 0; + color: #102445; +} +h2{ + margin-bottom: 30px; +} + +h5{ + text-transform: uppercase; + letter-spacing: 2px; + line-height: 1.21; + margin: 0; +} +a{ + line-height: 1.38; + letter-spacing: 0.2px; + text-decoration: none; + color: #102445; +} + + +a:hover{ + color: #1db598; +} + +textarea{ + width: 100%; + margin-bottom: 10px; + border-radius: 2px; + background-color: #fbfcfd; + border: solid 1.5px #e4eff4; + font-family: monospace; + font-size: 16px; + line-height: 1.5; + letter-spacing: 1.1px; + outline: none; + padding: 30px; +} + + +select{ + /*-webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background: url("img/chevron-down.png") no-repeat calc(100% - 10px) !important;*/ + border-radius: 3px; + background-color: #fbfcfd; + width: 187px; + height: 50px; + border: solid 1.5px #e4eff4; + font-size: 18px; + font-weight: bold; + line-height: 1.39; + letter-spacing: 0.2px; + color: #102445; + padding: 10.5px; + +} + +button{ + width: 165px; + height: 50px; + border-radius: 3px; + background-color: #1db598; + outline: none; + border: none; + font-size: 16px; + color: white; +} + +em{ + font-style: normal; + opacity: 0.5; + font-size: 14px; + font-weight: 500; + letter-spacing: 0.2px; + color: #102445; +} + +.header{ + height: 92px; + display: flex; + align-items: center; + justify-content: space-between; + width: 90%; + margin: auto; +} + +.container{ + display: flex; + height:100%; + min-height: 100vh; +} +.nav{ + color: white; + display: flex; + flex-direction: column; + width: 150px; + background: #3c5f8d; +} + +.nav-item{ + height:150px; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + border-bottom: 3px solid #2a4871; + cursor: pointer; +} + +.nav-active{ + background: #f2f7f9; + font-weight: bold; + color: #1db598; +} + +.nav-item > .nav-image { + display: flex; +} + +.nav-item > .nav-image-active { + display: none; +} + +.nav-active > .nav-image { + display: none; +} + +.nav-active > .nav-image-active { + display: flex; +} + + +.interface{ + display: none; + flex: 1; + flex-direction: column; + align-items: center; + background: #f2f7f9; +} + +.interface-active{ + display: flex; +} + +.query{ + margin-bottom: 30px; +} +.query, +.output-container, +.status-container{ + width: 75%; +} + +.output{ + margin-bottom: 30px; +} + +.input-controls{ + display: flex; + justify-content: flex-end; +} + +.tabs{ + display: flex; + background: #eaf2f6; +} +.active-tab{ + background: white; + font-weight: bold; + color: #1db598; + +} + +.tab{ + height:60px; + width: 100px; + border-top-right-radius: 5px; + display: flex; + align-items: center; + justify-content: center; + visibility: visible; + cursor: pointer; + +} + +.pane{ + background: white; + padding: 30px; + display: none; +} + +.active{ + display: block; +} + + + +.result-io-header{ + display: flex; + align-items: center; + margin-bottom: 15px; +} + +.result-input, +.result-output, +.result-error{ + border-radius: 2px; + background-color: #fafafa; + border: solid 1.5px #e4eff4; + font-family: monospace; + font-size: 16px; + line-height: 1.5; + letter-spacing: 1.1px; + color: #102445; + padding: 15px; + margin-bottom: 15px; +} + + +.result-output{ + background-color: #edf9f7; + border-left: solid 4px #1db598; +} + +.result-error{ + background-color: #fbf1f0; + border-left: solid 4px #fa3035; + color: #fa3035; +} + +.raw{ + height: 253px; + display: flex; + align-items: center; + justify-content: center; +} + + +.result-table > table { + border-left: solid 4px #1db598; +} + +table{ + border: solid 0.5px #e0e0e0; + width: 100%; + margin-bottom: 30px; + /*color:#3c5f8d;*/ +} +caption{ + text-align:left; + font-size: 16px; + font-weight: bold; + line-height: 1.21; + letter-spacing: 2px; + text-align: left; +} +th{ + font-size: 14px; + font-weight: bold; + line-height: 1.21; + letter-spacing: 2px; + color: #102445; + text-transform: uppercase; + text-align: left; + padding: 21px 30px; + background-color: white; +} +tr{ + border: solid 0.5px #e0e0e0; + background-color: white; +} +tr:nth-child(even) { + background-color: #f2f7f9; +} +td{ + padding: 21px 30px; +} + +.string { color: green; } +.number { color: darkorange; } +.boolean { color: blue; } +.null { color: magenta; } +.key { color: red; } \ No newline at end of file diff --git a/webui/index.html b/webui/index.html new file mode 100644 index 000000000..8422d4c45 --- /dev/null +++ b/webui/index.html @@ -0,0 +1,89 @@ + + + + + + + + Pilosa WebUI + + + +
+ +
+
+
+
+ + + +
+
+ +
+

Query

+ +
+ + +     + +
+
+ + +
+ +

Output

+
+ + +
+ +
+ +
+ +
+
+

Nodes

+
+
+
+
+

Indexes

+
+
+
+
+

Slice Distribution

+
+ ... +
+
+ +
+ +
+ docs! +
+ +
+ + +